diff --git a/.gitignore b/.gitignore
index 44bcb5db..12e1af62 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,35 +1,5 @@
-# Compiled Object files
-*.slo
-*.lo
-*.o
-*.obj
-
-# Precompiled Headers
-*.gch
-*.pch
-
-# Compiled Dynamic libraries
-*.so
-*.dylib
-*.dll
-
-# Fortran module files
-*.mod
-
-# Compiled Static libraries
-*.lai
-*.la
-*.a
-*.lib
-
-# Executables
-*.exe
-*.out
-*.app
-
-# Biicode directory
-bii
-bin
-
umltest.inner.sh
umltest.status
+/build
+/cmake
+/.idea
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 00000000..94a25f7f
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
index e769b2b7..fe851e2f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,17 +1,37 @@
language: cpp
sudo: required
+dist: trusty
compiler:
- gcc
+- clang
+os:
+- linux
+- osx
+matrix:
+ allow_failures:
+ - os: osx
+ - compiler: clang
+addons:
+ apt:
+ packages:
+ - libcrypto++-dev
+ - libfuse-dev
install:
-- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/update_gcc_version.sh
- && chmod +x update_gcc_version.sh
- && ./update_gcc_version.sh 4.8
- && rm update_gcc_version.sh
-- sudo apt-get install libfuse-dev
-# This is needed for packaging 7z distribution packages
-- sudo apt-get install software-properties-common && sudo add-apt-repository ppa:george-edison55/precise-backports -y && sudo apt-get update
-- sudo apt-get install cmake cmake-data rpm
-# CryFS needs cmake >= 3.3, install it.
+# Install boost
+- wget -O boost.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.56.0/boost_1_56_0.tar.bz2/download
+- tar -xf boost.tar.bz2
+- cd boost_1_56_0
+# TODO We should use clang as toolchain for building boost when clang is used for building our code
+- ./bootstrap.sh --with-libraries=filesystem,thread,chrono
+- sudo ./b2 -d0 install
+- cd ..
+- sudo rm -rf boost.tar.bz2 boost_1_56_0
+# Install run_with_fuse.sh
+- mkdir cmake
+- cd cmake
+- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/run_with_fuse.sh
+- chmod +x run_with_fuse.sh
+# Install cmake >= 3.3
- wget --no-check-certificate https://cmake.org/files/v3.3/cmake-3.3.2-Linux-x86_64.tar.gz
&& tar -xf cmake-3.3.2-Linux-x86_64.tar.gz
&& sudo cp -R cmake-3.3.2-Linux-x86_64/* /usr
@@ -19,35 +39,14 @@ install:
- cmake --version
# Use /dev/urandom when /dev/random is accessed, because travis doesn't have enough entropy
- sudo cp -a /dev/urandom /dev/random
-before_script:
-- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/setup_biicode_project.sh
- && chmod +x setup_biicode_project.sh
- && ./setup_biicode_project.sh
- && rm setup_biicode_project.sh
script:
-#The configure line is needed as a workaround for the following link, otherwise we wouldn't need "bii configure" at all because "bii build" calls it: http://forum.biicode.com/t/error-could-not-find-the-following-static-boost-libraries-boost-thread/374
-- bii cpp:configure || bii cpp:configure
-# Build cryfs executable
-- bii cpp:build -- -j2
-# Build and run test cases
-- bii cpp:build --target messmer_cryfs_test_main -- -j2
-- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/run_with_fuse.sh
- && chmod +x run_with_fuse.sh
- && ./run_with_fuse.sh "./bin/messmer_cryfs_test_main"
- && rm run_with_fuse.sh
-# Make distribution packages
-- bii clean
-- bii cpp:configure -D CMAKE_BUILD_TYPE=Release
-- bii build -- -j2
-- cd bii/build/messmer_cryfs && make package && cd ../../..
-after_success:
-- bii user ${BII_USERNAME} -p ${BII_PASSWORD}
-- bii publish
-#deploy:
-# provider: biicode
-# user: ${BII_USERNAME}
-# password:
-# secure: ${BII_PASSWORD}
-# on:
-# branch: develop
-
+- cmake ..
+- make -j2
+- make package -j2
+- ./test/cpp-utils/cpp-utils-test
+- ./run_with_fuse.sh ./test/fspp/fspp-test
+- ./test/parallelaccessstore/parallelaccessstore-test
+- ./test/blockstore/blockstore-test
+- ./test/blobstore/blobstore-test
+after_script:
+- rm run_with_fuse.sh
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c66790b2..86348775 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,26 +1,15 @@
# Earlier cmake versions generate .deb packages for which the package manager says they're bad quality
# and asks the user whether they really want to install it. Cmake 3.3 fixes this.
-CMAKE_MINIMUM_REQUIRED(VERSION 3.3)
+cmake_minimum_required(VERSION 3.3)
-INCLUDE(messmer/cmake/tools)
-INCLUDE(messmer/gitversion/cmake)
+include(utils.cmake)
-SETUP_GOOGLETEST()
+require_gcc_version(4.8)
-# Actually create targets: EXEcutables and libraries.
-ADD_BII_TARGETS()
-
-ACTIVATE_CPP14()
-
-ADD_BOOST(program_options chrono)
-
-ADD_DEFINITIONS(-D_FILE_OFFSET_BITS=64)
-
-GIT_VERSION_INIT()
-
-ENABLE_STYLE_WARNINGS()
-
-SET_TARGET_PROPERTIES(${BII_src_main_TARGET} PROPERTIES OUTPUT_NAME cryfs)
+add_subdirectory(vendor)
+add_subdirectory(gitversion)
+add_subdirectory(src)
+add_subdirectory(test)
# Fix debfiles permissions. Unfortunately, git doesn't store file permissions.
# When installing the .deb package and these files have the wrong permissions, the package manager complains.
@@ -64,88 +53,3 @@ SET(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/debfiles/pos
INCLUDE(CPack)
-# You can safely delete lines from here...
-
-###############################################################################
-# REFERENCE #
-###############################################################################
-#
-# This CMakeLists.txt file helps defining your block building and compiling
-# To learn more about the CMake use with biicode, visit http://docs.biicode.com/c++.html
-#
-# ----------------------------------------------------
-# NEW FEATURE! Include cmake files from remote blocks:
-# -----------------------------------------------------
-# Now you can handle cmake dependencies alike you do with c/c++:
-#
-# INCLUDE(user/block/myrecipe) # include myrecipe.cmake from remote user/block
-#
-# > EXAMPLE: Include our recipes and activate C++11 in your block (http://www.biicode.com/biicode/cmake)
-#
-# INCLUDE(biicode/cmake/tools) # Include tools.cmake file from "cmake" block from the "biicode" user
-# ACTIVATE_CPP11(INTERFACE ${BII_BLOCK_TARGET})
-#
-# Remember to run "bii find" to download out cmake tools file
-#
-# ---------------------
-# INIT_BIICODE_BLOCK()
-# ---------------------
-# This function creates several helper variables as ${BII_BLOCK_NAME} and ${BII_BLOCK_USER}
-# Also it loads variables from the cmake/bii_user_block_vars.cmake
-# ${BII_LIB_SRC} File list to create the library
-# ${BII_LIB_TYPE} Empty (default, STATIC most casess) STATIC or SHARED
-# ${BII_LIB_DEPS} Dependencies to other libraries (user2_block2, user3_blockX)
-# ${BII_LIB_SYSTEM_HEADERS} System linking requirements as windows.h, pthread.h, etc
-#
-# You can use or modify them here, for example, to add or remove files from targets based on OS
-# Or use typical cmake configurations done BEFORE defining targets. Examples:
-# ADD_DEFINITIONS(-DFOO)
-# FIND_PACKAGE(OpenGL QUIET)
-# You can add INCLUDE_DIRECTORIES here too
-#
-# ---------------------
-# ADD_BIICODE_TARGETS()
-# ---------------------
-#
-# This function creates the following variables:
-# ${BII_BLOCK_TARGET} Interface (no files) target for convenient configuration of all
-# targets in this block, as the rest of targets always depend on it
-# has name in the form "user_block_interface"
-# ${BII_LIB_TARGET} Target library name, usually in the form "user_block". May not exist
-# if BII_LIB_SRC is empty
-# ${BII_BLOCK_TARGETS} List of all targets defined in this block
-# ${BII_BLOCK_EXES} List of executables targets defined in this block
-# ${BII_exe_name_TARGET}: Executable target (e.g. ${BII_main_TARGET}. You can also use
-# directly the name of the executable target (e.g. user_block_main)
-#
-# > EXAMPLE: Add include directories to all targets of this block
-#
-# TARGET_INCLUDE_DIRECTORIES(${BII_BLOCK_TARGET} INTERFACE myincludedir)
-#
-# You can add private include directories to the Lib (if existing)
-#
-# > EXAMPLE: Link with pthread:
-#
-# TARGET_LINK_LIBRARIES(${BII_BLOCK_TARGET} INTERFACE pthread)
-# or link against library:
-# TARGET_LINK_LIBRARIES(${BII_LIB_TARGET} PUBLIC pthread)
-# or directly use the library target name:
-# TARGET_LINK_LIBRARIES(user_block PUBLIC pthread)
-#
-# NOTE: This can be also done adding pthread to ${BII_LIB_DEPS}
-# BEFORE calling ADD_BIICODE_TARGETS()
-#
-# > EXAMPLE: how to activate C++11
-#
-# IF(APPLE)
-# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11 -stdlib=libc++")
-# ELSEIF (WIN32 OR UNIX)
-# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11")
-# ENDIF(APPLE)
-#
-# > EXAMPLE: Set properties to target
-#
-# SET_TARGET_PROPERTIES(${BII_BLOCK_TARGET} PROPERTIES COMPILE_DEFINITIONS "IOV_MAX=255")
-#
-
-
diff --git a/CMakeLists.txt~ b/CMakeLists.txt~
new file mode 100644
index 00000000..3a0c8a4a
--- /dev/null
+++ b/CMakeLists.txt~
@@ -0,0 +1,165 @@
+<<<<<<< HEAD
+# Earlier cmake versions generate .deb packages for which the package manager says they're bad quality
+# and asks the user whether they really want to install it. Cmake 3.3 fixes this.
+CMAKE_MINIMUM_REQUIRED(VERSION 3.3)
+
+INCLUDE(messmer/cmake/tools)
+INCLUDE(messmer/gitversion/cmake)
+
+SETUP_GOOGLETEST()
+
+# Actually create targets: EXEcutables and libraries.
+ADD_BII_TARGETS()
+
+ACTIVATE_CPP14()
+
+ADD_BOOST(program_options chrono)
+
+ADD_DEFINITIONS(-D_FILE_OFFSET_BITS=64)
+
+GIT_VERSION_INIT()
+
+ENABLE_STYLE_WARNINGS()
+
+SET_TARGET_PROPERTIES(${BII_src_main_TARGET} PROPERTIES OUTPUT_NAME cryfs)
+
+# Fix debfiles permissions. Unfortunately, git doesn't store file permissions.
+# When installing the .deb package and these files have the wrong permissions, the package manager complains.
+EXECUTE_PROCESS(COMMAND /bin/bash -c "chmod 0755 ${CMAKE_CURRENT_SOURCE_DIR}/debfiles/*")
+
+INSTALL(TARGETS ${BII_src_main_TARGET}
+ DESTINATION bin
+ CONFIGURATIONS Release)
+
+SET(CPACK_GENERATOR TGZ DEB RPM)
+SET(CPACK_PACKAGE_NAME "cryfs")
+SET(CPACK_PACKAGE_VERSION "${GITVERSION_VERSION_STRING}")
+SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Encrypt your files and store them in the cloud.")
+SET(CPACK_PACKAGE_DESCRIPTION "CryFS encrypts your files, so you can safely store them anywhere. It works well together with cloud services like Dropbox, iCloud, OneDrive and others.")
+SET(CPACK_PACKAGE_CONTACT "Sebastian Messmer ")
+SET(CPACK_PACKAGE_VENDOR "Sebastian Messmer")
+SET(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE")
+SET(CPACK_PACKAGE_INSTALL_DIRECTORY "CMake ${CMake_VERSION_MAJOR}.${CMake_VERSION_MINOR}")
+IF(WIN32 AND NOT UNIX)
+ # There is a bug in NSI that does not handle full unix paths properly. Make
+ # sure there is at least one set of four (4) backlasshes.
+ #SET(CPACK_PACKAGE_ICON "${CMake_SOURCE_DIR}/Utilities/Release\\\\InstallIcon.bmp")
+ #SET(CPACK_NSIS_INSTALLED_ICON_NAME "bin\\\\cryfs.exe")
+ #SET(CPACK_NSIS_DISPLAY_NAME "${CPACK_PACKAGE_INSTALL_DIRECTORY} CryFS")
+ #SET(CPACK_NSIS_HELP_LINK "http:\\\\\\\\www.cryfs.org")
+ #SET(CPACK_NSIS_URL_INFO_ABOUT "http:\\\\\\\\www.cryfs.org")
+ #SET(CPACK_NSIS_CONTACT "messmer@cryfs.org")
+ #SET(CPACK_NSIS_MODIFY_PATH ON)
+ELSE(WIN32 AND NOT UNIX)
+ SET(CPACK_STRIP_FILES "bin/cryfs")
+ SET(CPACK_SOURCE_STRIP_FILES "")
+ENDIF(WIN32 AND NOT UNIX)
+SET(CPACK_PACKAGE_EXECUTABLES "cryfs" "CryFS")
+SET(CPACK_DEBIAN_PACKAGE_SECTION "utils")
+SET(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
+SET(CPACK_DEBIAN_PACKAGE_RECOMMENDS "fuse")
+
+SET(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://www.cryfs.org")
+SET(CPACK_RPM_PACKAGE_LICENSE "LGPLv3")
+SET(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/debfiles/postinst;${CMAKE_CURRENT_SOURCE_DIR}/debfiles/postrm")
+
+INCLUDE(CPack)
+
+# You can safely delete lines from here...
+
+###############################################################################
+# REFERENCE #
+###############################################################################
+#
+# This CMakeLists.txt file helps defining your block building and compiling
+# To learn more about the CMake use with biicode, visit http://docs.biicode.com/c++.html
+#
+# ----------------------------------------------------
+# NEW FEATURE! Include cmake files from remote blocks:
+# -----------------------------------------------------
+# Now you can handle cmake dependencies alike you do with c/c++:
+#
+# INCLUDE(user/block/myrecipe) # include myrecipe.cmake from remote user/block
+#
+# > EXAMPLE: Include our recipes and activate C++11 in your block (http://www.biicode.com/biicode/cmake)
+#
+# INCLUDE(biicode/cmake/tools) # Include tools.cmake file from "cmake" block from the "biicode" user
+# ACTIVATE_CPP11(INTERFACE ${BII_BLOCK_TARGET})
+#
+# Remember to run "bii find" to download out cmake tools file
+#
+# ---------------------
+# INIT_BIICODE_BLOCK()
+# ---------------------
+# This function creates several helper variables as ${BII_BLOCK_NAME} and ${BII_BLOCK_USER}
+# Also it loads variables from the cmake/bii_user_block_vars.cmake
+# ${BII_LIB_SRC} File list to create the library
+# ${BII_LIB_TYPE} Empty (default, STATIC most casess) STATIC or SHARED
+# ${BII_LIB_DEPS} Dependencies to other libraries (user2_block2, user3_blockX)
+# ${BII_LIB_SYSTEM_HEADERS} System linking requirements as windows.h, pthread.h, etc
+#
+# You can use or modify them here, for example, to add or remove files from targets based on OS
+# Or use typical cmake configurations done BEFORE defining targets. Examples:
+# ADD_DEFINITIONS(-DFOO)
+# FIND_PACKAGE(OpenGL QUIET)
+# You can add INCLUDE_DIRECTORIES here too
+#
+# ---------------------
+# ADD_BIICODE_TARGETS()
+# ---------------------
+#
+# This function creates the following variables:
+# ${BII_BLOCK_TARGET} Interface (no files) target for convenient configuration of all
+# targets in this block, as the rest of targets always depend on it
+# has name in the form "user_block_interface"
+# ${BII_LIB_TARGET} Target library name, usually in the form "user_block". May not exist
+# if BII_LIB_SRC is empty
+# ${BII_BLOCK_TARGETS} List of all targets defined in this block
+# ${BII_BLOCK_EXES} List of executables targets defined in this block
+# ${BII_exe_name_TARGET}: Executable target (e.g. ${BII_main_TARGET}. You can also use
+# directly the name of the executable target (e.g. user_block_main)
+#
+# > EXAMPLE: Add include directories to all targets of this block
+#
+# TARGET_INCLUDE_DIRECTORIES(${BII_BLOCK_TARGET} INTERFACE myincludedir)
+#
+# You can add private include directories to the Lib (if existing)
+#
+# > EXAMPLE: Link with pthread:
+#
+# TARGET_LINK_LIBRARIES(${BII_BLOCK_TARGET} INTERFACE pthread)
+# or link against library:
+# TARGET_LINK_LIBRARIES(${BII_LIB_TARGET} PUBLIC pthread)
+# or directly use the library target name:
+# TARGET_LINK_LIBRARIES(user_block PUBLIC pthread)
+#
+# NOTE: This can be also done adding pthread to ${BII_LIB_DEPS}
+# BEFORE calling ADD_BIICODE_TARGETS()
+#
+# > EXAMPLE: how to activate C++11
+#
+# IF(APPLE)
+# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11 -stdlib=libc++")
+# ELSEIF (WIN32 OR UNIX)
+# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11")
+# ENDIF(APPLE)
+#
+# > EXAMPLE: Set properties to target
+#
+# SET_TARGET_PROPERTIES(${BII_BLOCK_TARGET} PROPERTIES COMPILE_DEFINITIONS "IOV_MAX=255")
+#
+
+
+=======
+# Earlier cmake versions generate .deb packages for which the package manager says they're bad quality
+# and asks the user whether they really want to install it. Cmake 3.3 fixes this.
+cmake_minimum_required(VERSION 3.3)
+
+include(utils.cmake)
+
+require_gcc_version(4.8)
+
+add_subdirectory(vendor)
+add_subdirectory(src)
+add_subdirectory(test)
+>>>>>>> cpu/cmake
diff --git a/README.md b/README.md
index d8430e93..d192164e 100644
--- a/README.md
+++ b/README.md
@@ -51,7 +51,7 @@ Requirements
- GCC version >= 4.8 or Clang (TODO which minimal version?)
- CMake version >= 3.3
- libcurl4 (including development headers)
- - Boost libraries filesystem, system, chrono, thread in version >= 1.56
+ - Boost libraries filesystem, system, chrono, program_options, thread in version >= 1.56
- Crypto++ >= 5.6.3 (TODO Lower minimal version possible?)
- libFUSE >= 2.8.6 (including development headers)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
new file mode 100644
index 00000000..5b5b648f
--- /dev/null
+++ b/src/CMakeLists.txt
@@ -0,0 +1,7 @@
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+add_subdirectory(cpp-utils)
+add_subdirectory(fspp)
+add_subdirectory(blockstore)
+add_subdirectory(blobstore)
+add_subdirectory(cryfs)
\ No newline at end of file
diff --git a/src/blobstore/CMakeLists.txt b/src/blobstore/CMakeLists.txt
new file mode 100644
index 00000000..141e33be
--- /dev/null
+++ b/src/blobstore/CMakeLists.txt
@@ -0,0 +1,29 @@
+project (blobstore)
+
+set(SOURCES
+ implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.cpp
+ implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.cpp
+ implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp
+ implementations/onblocks/utils/Math.cpp
+ implementations/onblocks/BlobStoreOnBlocks.cpp
+ implementations/onblocks/datanodestore/DataNode.cpp
+ implementations/onblocks/datanodestore/DataLeafNode.cpp
+ implementations/onblocks/datanodestore/DataInnerNode.cpp
+ implementations/onblocks/datanodestore/DataNodeStore.cpp
+ implementations/onblocks/datatreestore/impl/algorithms.cpp
+ implementations/onblocks/datatreestore/DataTree.cpp
+ implementations/onblocks/datatreestore/DataTreeStore.cpp
+ implementations/onblocks/BlobOnBlocks.cpp
+)
+
+add_library(${PROJECT_NAME} STATIC ${SOURCES})
+
+# This is needed by boost thread
+if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+ target_link_libraries(${PROJECT_NAME} PRIVATE rt)
+endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+target_link_libraries(${PROJECT_NAME} PUBLIC cpp-utils blockstore)
+
+target_add_boost(${PROJECT_NAME} filesystem system thread)
+target_enable_style_warnings(${PROJECT_NAME})
+target_activate_cpp14(${PROJECT_NAME})
diff --git a/src/blobstore/implementations/onblocks/BlobOnBlocks.cpp b/src/blobstore/implementations/onblocks/BlobOnBlocks.cpp
new file mode 100644
index 00000000..174183ca
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/BlobOnBlocks.cpp
@@ -0,0 +1,109 @@
+#include "parallelaccessdatatreestore/DataTreeRef.h"
+#include "BlobOnBlocks.h"
+
+#include "datanodestore/DataLeafNode.h"
+#include "utils/Math.h"
+#include
+#include
+
+using std::function;
+using cpputils::unique_ref;
+using cpputils::Data;
+using blobstore::onblocks::datanodestore::DataLeafNode;
+using blobstore::onblocks::datanodestore::DataNodeLayout;
+using blockstore::Key;
+
+namespace blobstore {
+namespace onblocks {
+
+using parallelaccessdatatreestore::DataTreeRef;
+
+BlobOnBlocks::BlobOnBlocks(unique_ref datatree)
+: _datatree(std::move(datatree)), _sizeCache(boost::none) {
+}
+
+BlobOnBlocks::~BlobOnBlocks() {
+}
+
+uint64_t BlobOnBlocks::size() const {
+ if (_sizeCache == boost::none) {
+ _sizeCache = _datatree->numStoredBytes();
+ }
+ return *_sizeCache;
+}
+
+void BlobOnBlocks::resize(uint64_t numBytes) {
+ _datatree->resizeNumBytes(numBytes);
+ _sizeCache = numBytes;
+}
+
+void BlobOnBlocks::traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function func) const {
+ uint64_t endByte = beginByte + sizeBytes;
+ uint32_t firstLeaf = beginByte / _datatree->maxBytesPerLeaf();
+ uint32_t endLeaf = utils::ceilDivision(endByte, _datatree->maxBytesPerLeaf());
+ bool writingOutside = size() < endByte; // TODO Calling size() is slow because it has to traverse the tree
+ _datatree->traverseLeaves(firstLeaf, endLeaf, [&func, beginByte, endByte, endLeaf, writingOutside](DataLeafNode *leaf, uint32_t leafIndex) {
+ uint64_t indexOfFirstLeafByte = leafIndex * leaf->maxStoreableBytes();
+ uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
+ uint32_t dataEnd = std::min(leaf->maxStoreableBytes(), endByte - indexOfFirstLeafByte);
+ if (leafIndex == endLeaf-1 && writingOutside) {
+ // If we are traversing an area that didn't exist before, then the last leaf was just created with a wrong size. We have to fix it.
+ leaf->resize(dataEnd);
+ }
+ func(indexOfFirstLeafByte, leaf, dataBegin, dataEnd-dataBegin);
+ });
+ if (writingOutside) {
+ ASSERT(_datatree->numStoredBytes() == endByte, "Writing didn't grow by the correct number of bytes");
+ _sizeCache = endByte;
+ }
+}
+
+Data BlobOnBlocks::readAll() const {
+ //TODO Querying size is inefficient. Is this possible without a call to size()?
+ uint64_t count = size();
+ Data result(count);
+ _read(result.data(), 0, count);
+ return result;
+}
+
+void BlobOnBlocks::read(void *target, uint64_t offset, uint64_t count) const {
+ ASSERT(offset <= size() && offset + count <= size(), "BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
+ uint64_t read = tryRead(target, offset, count);
+ ASSERT(read == count, "BlobOnBlocks::read() couldn't read all requested bytes. Use BlobOnBlocks::tryRead() if this should be allowed.");
+}
+
+uint64_t BlobOnBlocks::tryRead(void *target, uint64_t offset, uint64_t count) const {
+ //TODO Quite inefficient to call size() here, because that has to traverse the tree
+ uint64_t realCount = std::max(UINT64_C(0), std::min(count, size()-offset));
+ _read(target, offset, realCount);
+ return realCount;
+}
+
+void BlobOnBlocks::_read(void *target, uint64_t offset, uint64_t count) const {
+ traverseLeaves(offset, count, [target, offset] (uint64_t indexOfFirstLeafByte, const DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
+ //TODO Simplify formula, make it easier to understand
+ leaf->read((uint8_t*)target + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
+ });
+}
+
+void BlobOnBlocks::write(const void *source, uint64_t offset, uint64_t count) {
+ traverseLeaves(offset, count, [source, offset] (uint64_t indexOfFirstLeafByte, DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
+ //TODO Simplify formula, make it easier to understand
+ leaf->write((uint8_t*)source + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
+ });
+}
+
+void BlobOnBlocks::flush() {
+ _datatree->flush();
+}
+
+const Key &BlobOnBlocks::key() const {
+ return _datatree->key();
+}
+
+unique_ref BlobOnBlocks::releaseTree() {
+ return std::move(_datatree);
+}
+
+}
+}
diff --git a/src/blobstore/implementations/onblocks/BlobOnBlocks.h b/src/blobstore/implementations/onblocks/BlobOnBlocks.h
new file mode 100644
index 00000000..7d73db39
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/BlobOnBlocks.h
@@ -0,0 +1,52 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_BLOBONBLOCKS_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_BLOBONBLOCKS_H_
+
+#include "../../interface/Blob.h"
+
+#include
+#include
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+class DataLeafNode;
+}
+namespace parallelaccessdatatreestore {
+class DataTreeRef;
+}
+
+class BlobOnBlocks final: public Blob {
+public:
+ BlobOnBlocks(cpputils::unique_ref datatree);
+ ~BlobOnBlocks();
+
+ const blockstore::Key &key() const override;
+
+ uint64_t size() const override;
+ void resize(uint64_t numBytes) override;
+
+ cpputils::Data readAll() const override;
+ void read(void *target, uint64_t offset, uint64_t size) const override;
+ uint64_t tryRead(void *target, uint64_t offset, uint64_t size) const override;
+ void write(const void *source, uint64_t offset, uint64_t size) override;
+
+ void flush() override;
+
+ cpputils::unique_ref releaseTree();
+
+private:
+
+ void _read(void *target, uint64_t offset, uint64_t count) const;
+ void traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function) const;
+
+ cpputils::unique_ref _datatree;
+ mutable boost::optional _sizeCache;
+
+ DISALLOW_COPY_AND_ASSIGN(BlobOnBlocks);
+};
+
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp b/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp
new file mode 100644
index 00000000..588a94cf
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp
@@ -0,0 +1,56 @@
+#include "parallelaccessdatatreestore/DataTreeRef.h"
+#include "parallelaccessdatatreestore/ParallelAccessDataTreeStore.h"
+#include
+#include "datanodestore/DataLeafNode.h"
+#include "datanodestore/DataNodeStore.h"
+#include "datatreestore/DataTreeStore.h"
+#include "datatreestore/DataTree.h"
+#include "BlobStoreOnBlocks.h"
+#include "BlobOnBlocks.h"
+#include
+#include
+
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+
+using blockstore::BlockStore;
+using blockstore::parallelaccess::ParallelAccessBlockStore;
+using blockstore::Key;
+using cpputils::dynamic_pointer_move;
+using boost::optional;
+using boost::none;
+
+namespace blobstore {
+namespace onblocks {
+
+using datanodestore::DataNodeStore;
+using datatreestore::DataTreeStore;
+using parallelaccessdatatreestore::ParallelAccessDataTreeStore;
+
+BlobStoreOnBlocks::BlobStoreOnBlocks(unique_ref blockStore, uint32_t blocksizeBytes)
+: _dataTreeStore(make_unique_ref(make_unique_ref(make_unique_ref(make_unique_ref(std::move(blockStore)), blocksizeBytes)))) {
+}
+
+BlobStoreOnBlocks::~BlobStoreOnBlocks() {
+}
+
+unique_ref BlobStoreOnBlocks::create() {
+ return make_unique_ref(_dataTreeStore->createNewTree());
+}
+
+optional> BlobStoreOnBlocks::load(const Key &key) {
+ auto tree = _dataTreeStore->load(key);
+ if (tree == none) {
+ return none;
+ }
+ return optional>(make_unique_ref(std::move(*tree)));
+}
+
+void BlobStoreOnBlocks::remove(unique_ref blob) {
+ auto _blob = dynamic_pointer_move(blob);
+ ASSERT(_blob != none, "Passed Blob in BlobStoreOnBlocks::remove() is not a BlobOnBlocks.");
+ _dataTreeStore->remove((*_blob)->releaseTree());
+}
+
+}
+}
diff --git a/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h b/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h
new file mode 100644
index 00000000..0dcea458
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h
@@ -0,0 +1,35 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_BLOCKED_BLOBSTOREONBLOCKS_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_BLOCKED_BLOBSTOREONBLOCKS_H_
+
+#include "../../interface/BlobStore.h"
+#include
+
+namespace blobstore {
+namespace onblocks {
+namespace parallelaccessdatatreestore {
+class ParallelAccessDataTreeStore;
+}
+
+//TODO Make blobstore able to cope with incomplete data (some blocks missing, because they're not synchronized yet) and write test cases for that
+
+class BlobStoreOnBlocks final: public BlobStore {
+public:
+ BlobStoreOnBlocks(cpputils::unique_ref blockStore, uint32_t blocksizeBytes);
+ ~BlobStoreOnBlocks();
+
+ cpputils::unique_ref create() override;
+ boost::optional> load(const blockstore::Key &key) override;
+
+ void remove(cpputils::unique_ref blob) override;
+
+private:
+ cpputils::unique_ref _dataTreeStore;
+
+ DISALLOW_COPY_AND_ASSIGN(BlobStoreOnBlocks);
+};
+
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp b/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp
new file mode 100644
index 00000000..dda86ece
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp
@@ -0,0 +1,87 @@
+#include "DataInnerNode.h"
+#include "DataNodeStore.h"
+#include
+
+using blockstore::Block;
+using cpputils::Data;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using blockstore::Key;
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+
+DataInnerNode::DataInnerNode(DataNodeView view)
+: DataNode(std::move(view)) {
+ ASSERT(depth() > 0, "Inner node can't have depth 0. Is this a leaf maybe?");
+}
+
+DataInnerNode::~DataInnerNode() {
+}
+
+unique_ref DataInnerNode::InitializeNewNode(unique_ref block, const DataNode &first_child) {
+ DataNodeView node(std::move(block));
+ node.setDepth(first_child.depth() + 1);
+ node.setSize(1);
+ auto result = make_unique_ref(std::move(node));
+ result->ChildrenBegin()->setKey(first_child.key());
+ return result;
+}
+
+uint32_t DataInnerNode::numChildren() const {
+ return node().Size();
+}
+
+DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() {
+ return const_cast(const_cast(this)->ChildrenBegin());
+}
+
+const DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() const {
+ return node().DataBegin();
+}
+
+DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() {
+ return const_cast(const_cast(this)->ChildrenEnd());
+}
+
+const DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() const {
+ return ChildrenBegin() + node().Size();
+}
+
+DataInnerNode::ChildEntry *DataInnerNode::LastChild() {
+ return const_cast(const_cast(this)->LastChild());
+}
+
+const DataInnerNode::ChildEntry *DataInnerNode::LastChild() const {
+ return getChild(numChildren()-1);
+}
+
+DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) {
+ return const_cast(const_cast(this)->getChild(index));
+}
+
+const DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) const {
+ ASSERT(index < numChildren(), "Accessing child out of range");
+ return ChildrenBegin()+index;
+}
+
+void DataInnerNode::addChild(const DataNode &child) {
+ ASSERT(numChildren() < maxStoreableChildren(), "Adding more children than we can store");
+ ASSERT(child.depth() == depth()-1, "The child that should be added has wrong depth");
+ node().setSize(node().Size()+1);
+ LastChild()->setKey(child.key());
+}
+
+void DataInnerNode::removeLastChild() {
+ ASSERT(node().Size() > 1, "There is no child to remove");
+ node().setSize(node().Size()-1);
+}
+
+uint32_t DataInnerNode::maxStoreableChildren() const {
+ return node().layout().maxChildrenPerInnerNode();
+}
+
+}
+}
+}
diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.h b/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.h
new file mode 100644
index 00000000..967d6746
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.h
@@ -0,0 +1,49 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_H_
+
+#include "DataNode.h"
+#include "DataInnerNode_ChildEntry.h"
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+
+class DataInnerNode final: public DataNode {
+public:
+ static cpputils::unique_ref InitializeNewNode(cpputils::unique_ref block, const DataNode &first_child_key);
+
+ DataInnerNode(DataNodeView block);
+ ~DataInnerNode();
+
+ using ChildEntry = DataInnerNode_ChildEntry;
+
+ uint32_t maxStoreableChildren() const;
+
+ ChildEntry *getChild(unsigned int index);
+ const ChildEntry *getChild(unsigned int index) const;
+
+ uint32_t numChildren() const;
+
+ void addChild(const DataNode &child_key);
+
+ void removeLastChild();
+
+ ChildEntry *LastChild();
+ const ChildEntry *LastChild() const;
+
+private:
+
+ ChildEntry *ChildrenBegin();
+ ChildEntry *ChildrenEnd();
+ const ChildEntry *ChildrenBegin() const;
+ const ChildEntry *ChildrenEnd() const;
+
+ DISALLOW_COPY_AND_ASSIGN(DataInnerNode);
+};
+
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode_ChildEntry.h b/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode_ChildEntry.h
new file mode 100644
index 00000000..51e7afc1
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode_ChildEntry.h
@@ -0,0 +1,30 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_CHILDENTRY_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_CHILDENTRY_H_
+
+#include
+
+namespace blobstore{
+namespace onblocks{
+namespace datanodestore{
+
+struct DataInnerNode_ChildEntry final {
+public:
+ blockstore::Key key() const {
+ return blockstore::Key::FromBinary(_keydata);
+ }
+private:
+ void setKey(const blockstore::Key &key) {
+ key.ToBinary(_keydata);
+ }
+ friend class DataInnerNode;
+ uint8_t _keydata[blockstore::Key::BINARY_LENGTH];
+
+ DISALLOW_COPY_AND_ASSIGN(DataInnerNode_ChildEntry);
+};
+
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.cpp b/src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.cpp
new file mode 100644
index 00000000..7e3511ea
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.cpp
@@ -0,0 +1,67 @@
+#include "DataLeafNode.h"
+#include "DataInnerNode.h"
+#include
+
+using blockstore::Block;
+using cpputils::Data;
+using blockstore::Key;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+
+DataLeafNode::DataLeafNode(DataNodeView view)
+: DataNode(std::move(view)) {
+ ASSERT(node().Depth() == 0, "Leaf node must have depth 0. Is it an inner node instead?");
+ ASSERT(numBytes() <= maxStoreableBytes(), "Leaf says it stores more bytes than it has space for");
+}
+
+DataLeafNode::~DataLeafNode() {
+}
+
+unique_ref DataLeafNode::InitializeNewNode(unique_ref block) {
+ DataNodeView node(std::move(block));
+ node.setDepth(0);
+ node.setSize(0);
+ //fillDataWithZeroes(); not needed, because a newly created block will be zeroed out. DataLeafNodeTest.SpaceIsZeroFilledWhenGrowing ensures this.
+ return make_unique_ref(std::move(node));
+}
+
+void DataLeafNode::read(void *target, uint64_t offset, uint64_t size) const {
+ ASSERT(offset <= node().Size() && offset + size <= node().Size(), "Read out of valid area"); // Also check offset, because the addition could lead to overflows
+ std::memcpy(target, (uint8_t*)node().data() + offset, size);
+}
+
+void DataLeafNode::write(const void *source, uint64_t offset, uint64_t size) {
+ ASSERT(offset <= node().Size() && offset + size <= node().Size(), "Write out of valid area"); // Also check offset, because the addition could lead to overflows
+ node().write(source, offset, size);
+}
+
+uint32_t DataLeafNode::numBytes() const {
+ return node().Size();
+}
+
+void DataLeafNode::resize(uint32_t new_size) {
+ ASSERT(new_size <= maxStoreableBytes(), "Trying to resize to a size larger than the maximal size");
+ uint32_t old_size = node().Size();
+ if (new_size < old_size) {
+ fillDataWithZeroesFromTo(new_size, old_size);
+ }
+ node().setSize(new_size);
+}
+
+void DataLeafNode::fillDataWithZeroesFromTo(off_t begin, off_t end) {
+ Data ZEROES(end-begin);
+ ZEROES.FillWithZeroes();
+ node().write(ZEROES.data(), begin, end-begin);
+}
+
+uint64_t DataLeafNode::maxStoreableBytes() const {
+ return node().layout().maxBytesPerLeaf();
+}
+
+}
+}
+}
diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.h b/src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.h
new file mode 100644
index 00000000..d707eb44
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.h
@@ -0,0 +1,39 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATALEAFNODE_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATALEAFNODE_H_
+
+#include "DataNode.h"
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+class DataInnerNode;
+
+class DataLeafNode final: public DataNode {
+public:
+ static cpputils::unique_ref InitializeNewNode(cpputils::unique_ref block);
+
+ DataLeafNode(DataNodeView block);
+ ~DataLeafNode();
+
+ //Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
+ uint64_t maxStoreableBytes() const;
+
+ void read(void *target, uint64_t offset, uint64_t size) const;
+ void write(const void *source, uint64_t offset, uint64_t size);
+
+ uint32_t numBytes() const;
+
+ void resize(uint32_t size);
+
+private:
+ void fillDataWithZeroesFromTo(off_t begin, off_t end);
+
+ DISALLOW_COPY_AND_ASSIGN(DataLeafNode);
+};
+
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataNode.cpp b/src/blobstore/implementations/onblocks/datanodestore/DataNode.cpp
new file mode 100644
index 00000000..3b409e4f
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datanodestore/DataNode.cpp
@@ -0,0 +1,54 @@
+#include "DataInnerNode.h"
+#include "DataLeafNode.h"
+#include "DataNode.h"
+#include "DataNodeStore.h"
+#include
+
+using blockstore::Block;
+using blockstore::Key;
+
+using std::runtime_error;
+using cpputils::unique_ref;
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+
+DataNode::DataNode(DataNodeView node)
+: _node(std::move(node)) {
+}
+
+DataNode::~DataNode() {
+}
+
+DataNodeView &DataNode::node() {
+ return const_cast(const_cast(this)->node());
+}
+
+const DataNodeView &DataNode::node() const {
+ return _node;
+}
+
+const Key &DataNode::key() const {
+ return _node.key();
+}
+
+uint8_t DataNode::depth() const {
+ return _node.Depth();
+}
+
+unique_ref DataNode::convertToNewInnerNode(unique_ref node, const DataNode &first_child) {
+ Key key = node->key();
+ auto block = node->_node.releaseBlock();
+ blockstore::utils::fillWithZeroes(block.get());
+
+ return DataInnerNode::InitializeNewNode(std::move(block), first_child);
+}
+
+void DataNode::flush() const {
+ _node.flush();
+}
+
+}
+}
+}
diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataNode.h b/src/blobstore/implementations/onblocks/datanodestore/DataNode.h
new file mode 100644
index 00000000..e37c5415
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datanodestore/DataNode.h
@@ -0,0 +1,44 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODE_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODE_H_
+
+#include "DataNodeView.h"
+#include
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+class DataNodeStore;
+class DataInnerNode;
+
+class DataNode {
+public:
+ virtual ~DataNode();
+
+ const blockstore::Key &key() const;
+
+ uint8_t depth() const;
+
+ static cpputils::unique_ref convertToNewInnerNode(cpputils::unique_ref node, const DataNode &first_child);
+
+ void flush() const;
+
+protected:
+ DataNode(DataNodeView block);
+
+ DataNodeView &node();
+ const DataNodeView &node() const;
+ friend class DataNodeStore;
+
+private:
+ DataNodeView _node;
+
+ DISALLOW_COPY_AND_ASSIGN(DataNode);
+};
+
+}
+}
+}
+
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp
new file mode 100644
index 00000000..6849ead3
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp
@@ -0,0 +1,113 @@
+#include "DataInnerNode.h"
+#include "DataLeafNode.h"
+#include "DataNodeStore.h"
+#include
+#include
+#include
+#include
+
+using blockstore::BlockStore;
+using blockstore::Block;
+using blockstore::Key;
+using cpputils::Data;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using std::runtime_error;
+using boost::optional;
+using boost::none;
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+
+DataNodeStore::DataNodeStore(unique_ref blockstore, uint32_t blocksizeBytes)
+: _blockstore(std::move(blockstore)), _layout(blocksizeBytes) {
+}
+
+DataNodeStore::~DataNodeStore() {
+}
+
+unique_ref DataNodeStore::load(unique_ref block) {
+ ASSERT(block->size() == _layout.blocksizeBytes(), "Loading block of wrong size");
+ DataNodeView node(std::move(block));
+
+ if (node.Depth() == 0) {
+ return make_unique_ref(std::move(node));
+ } else if (node.Depth() <= MAX_DEPTH) {
+ return make_unique_ref(std::move(node));
+ } else {
+ throw runtime_error("Tree is to deep. Data corruption?");
+ }
+}
+
+unique_ref DataNodeStore::createNewInnerNode(const DataNode &first_child) {
+ ASSERT(first_child.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
+ //TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
+ auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
+ return DataInnerNode::InitializeNewNode(std::move(block), first_child);
+}
+
+unique_ref DataNodeStore::createNewLeafNode() {
+ //TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
+ auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
+ return DataLeafNode::InitializeNewNode(std::move(block));
+}
+
+optional> DataNodeStore::load(const Key &key) {
+ auto block = _blockstore->load(key);
+ if (block == none) {
+ return none;
+ } else {
+ return load(std::move(*block));
+ }
+}
+
+unique_ref DataNodeStore::createNewNodeAsCopyFrom(const DataNode &source) {
+ ASSERT(source.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
+ auto newBlock = blockstore::utils::copyToNewBlock(_blockstore.get(), source.node().block());
+ return load(std::move(newBlock));
+}
+
+unique_ref DataNodeStore::overwriteNodeWith(unique_ref target, const DataNode &source) {
+ ASSERT(target->node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Target node has wrong layout. Is it from the same DataNodeStore?");
+ ASSERT(source.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
+ Key key = target->key();
+ {
+ auto targetBlock = target->node().releaseBlock();
+ cpputils::destruct(std::move(target)); // Call destructor
+ blockstore::utils::copyTo(targetBlock.get(), source.node().block());
+ }
+ auto loaded = load(key);
+ ASSERT(loaded != none, "Couldn't load the target node after overwriting it");
+ return std::move(*loaded);
+}
+
+void DataNodeStore::remove(unique_ref node) {
+ auto block = node->node().releaseBlock();
+ cpputils::destruct(std::move(node)); // Call destructor
+ _blockstore->remove(std::move(block));
+}
+
+uint64_t DataNodeStore::numNodes() const {
+ return _blockstore->numBlocks();
+}
+
+void DataNodeStore::removeSubtree(unique_ref node) {
+ DataInnerNode *inner = dynamic_cast(node.get());
+ if (inner != nullptr) {
+ for (uint32_t i = 0; i < inner->numChildren(); ++i) {
+ auto child = load(inner->getChild(i)->key());
+ ASSERT(child != none, "Couldn't load child node");
+ removeSubtree(std::move(*child));
+ }
+ }
+ remove(std::move(node));
+}
+
+DataNodeLayout DataNodeStore::layout() const {
+ return _layout;
+}
+
+}
+}
+}
diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h
new file mode 100644
index 00000000..dc2bac44
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h
@@ -0,0 +1,60 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODESTORE_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODESTORE_H_
+
+#include
+#include
+#include "DataNodeView.h"
+#include
+
+namespace blockstore{
+class Block;
+class BlockStore;
+}
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+class DataNode;
+class DataLeafNode;
+class DataInnerNode;
+
+class DataNodeStore final {
+public:
+ DataNodeStore(cpputils::unique_ref blockstore, uint32_t blocksizeBytes);
+ ~DataNodeStore();
+
+ static constexpr uint8_t MAX_DEPTH = 10;
+
+ DataNodeLayout layout() const;
+
+ boost::optional> load(const blockstore::Key &key);
+
+ cpputils::unique_ref createNewLeafNode();
+ cpputils::unique_ref createNewInnerNode(const DataNode &first_child);
+
+ cpputils::unique_ref createNewNodeAsCopyFrom(const DataNode &source);
+
+ cpputils::unique_ref overwriteNodeWith(cpputils::unique_ref target, const DataNode &source);
+
+ void remove(cpputils::unique_ref node);
+
+ void removeSubtree(cpputils::unique_ref node);
+
+ uint64_t numNodes() const;
+ //TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree()
+
+private:
+ cpputils::unique_ref load(cpputils::unique_ref block);
+
+ cpputils::unique_ref _blockstore;
+ const DataNodeLayout _layout;
+
+ DISALLOW_COPY_AND_ASSIGN(DataNodeStore);
+};
+
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h b/src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h
new file mode 100644
index 00000000..7dfcb1c4
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h
@@ -0,0 +1,140 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODEVIEW_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODEVIEW_H_
+
+#include
+#include "../BlobStoreOnBlocks.h"
+#include "DataInnerNode_ChildEntry.h"
+
+#include
+
+#include
+#include
+#include
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+
+//TODO Move DataNodeLayout into own file
+class DataNodeLayout final {
+public:
+ constexpr DataNodeLayout(uint32_t blocksizeBytes)
+ :_blocksizeBytes(
+ (HEADERSIZE_BYTES + 2*sizeof(DataInnerNode_ChildEntry) <= blocksizeBytes)
+ ? blocksizeBytes
+ : throw std::logic_error("Blocksize too small, not enough space to store two children in an inner node")) {
+ }
+
+ //Total size of the header
+ static constexpr uint32_t HEADERSIZE_BYTES = 8;
+ //Where in the header is the depth field
+ static constexpr uint32_t DEPTH_OFFSET_BYTES = 0;
+ //Where in the header is the size field (for inner nodes: number of children, for leafs: content data size)
+ static constexpr uint32_t SIZE_OFFSET_BYTES = 4;
+
+
+ //Size of a block (header + data region)
+ constexpr uint32_t blocksizeBytes() const {
+ return _blocksizeBytes;
+ }
+
+ //Number of bytes in the data region of a node
+ constexpr uint32_t datasizeBytes() const {
+ return _blocksizeBytes - HEADERSIZE_BYTES;
+ }
+
+ //Maximum number of children an inner node can store
+ constexpr uint32_t maxChildrenPerInnerNode() const {
+ return datasizeBytes() / sizeof(DataInnerNode_ChildEntry);
+ }
+
+ //Maximum number of bytes a leaf can store
+ //We are returning uint64_t here, because calculations involving maxBytesPerLeaf most probably should use 64bit integers to support blobs >4GB.
+ constexpr uint64_t maxBytesPerLeaf() const {
+ return datasizeBytes();
+ }
+private:
+ uint32_t _blocksizeBytes;
+};
+
+class DataNodeView final {
+public:
+ DataNodeView(cpputils::unique_ref block): _block(std::move(block)) {
+ }
+ ~DataNodeView() {}
+
+ DataNodeView(DataNodeView &&rhs) = default;
+
+ uint8_t Depth() const {
+ return *((uint8_t*)_block->data()+DataNodeLayout::DEPTH_OFFSET_BYTES);
+ }
+
+ void setDepth(uint8_t value) {
+ _block->write(&value, DataNodeLayout::DEPTH_OFFSET_BYTES, sizeof(value));
+ }
+
+ uint32_t Size() const {
+ return *(uint32_t*)((uint8_t*)_block->data()+DataNodeLayout::SIZE_OFFSET_BYTES);
+ }
+
+ void setSize(uint32_t value) {
+ _block->write(&value, DataNodeLayout::SIZE_OFFSET_BYTES, sizeof(value));
+ }
+
+ const void *data() const {
+ return (uint8_t*)_block->data() + DataNodeLayout::HEADERSIZE_BYTES;
+ }
+
+ void write(const void *source, uint64_t offset, uint64_t size) {
+ _block->write(source, offset + DataNodeLayout::HEADERSIZE_BYTES, size);
+ }
+
+ template
+ const Entry *DataBegin() const {
+ return GetOffset();
+ }
+
+ template
+ const Entry *DataEnd() const {
+ const unsigned int NUM_ENTRIES = layout().datasizeBytes() / sizeof(Entry);
+ return DataBegin() + NUM_ENTRIES;
+ }
+
+ DataNodeLayout layout() const {
+ return DataNodeLayout(_block->size());
+ }
+
+ cpputils::unique_ref releaseBlock() {
+ return std::move(_block);
+ }
+
+ const blockstore::Block &block() const {
+ return *_block;
+ }
+
+ const blockstore::Key &key() const {
+ return _block->key();
+ }
+
+ void flush() const {
+ _block->flush();
+ }
+
+private:
+ template
+ const Type *GetOffset() const {
+ return (Type*)(((const int8_t*)_block->data())+offset);
+ }
+
+ cpputils::unique_ref _block;
+
+ DISALLOW_COPY_AND_ASSIGN(DataNodeView);
+
+};
+
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp b/src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp
new file mode 100644
index 00000000..d2d87049
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp
@@ -0,0 +1,334 @@
+#include "DataTree.h"
+
+#include "../datanodestore/DataNodeStore.h"
+#include "../datanodestore/DataInnerNode.h"
+#include "../datanodestore/DataLeafNode.h"
+#include "../utils/Math.h"
+
+#include "impl/algorithms.h"
+
+#include
+#include
+#include
+#include
+
+using blockstore::Key;
+using blobstore::onblocks::datanodestore::DataNodeStore;
+using blobstore::onblocks::datanodestore::DataNode;
+using blobstore::onblocks::datanodestore::DataInnerNode;
+using blobstore::onblocks::datanodestore::DataLeafNode;
+using blobstore::onblocks::datanodestore::DataNodeLayout;
+
+using std::dynamic_pointer_cast;
+using std::function;
+using boost::shared_mutex;
+using boost::shared_lock;
+using boost::unique_lock;
+using boost::none;
+using std::vector;
+
+using cpputils::dynamic_pointer_move;
+using cpputils::optional_ownership_ptr;
+using cpputils::WithOwnership;
+using cpputils::WithoutOwnership;
+using cpputils::unique_ref;
+
+namespace blobstore {
+namespace onblocks {
+namespace datatreestore {
+
+DataTree::DataTree(DataNodeStore *nodeStore, unique_ref rootNode)
+ : _mutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)) {
+}
+
+DataTree::~DataTree() {
+}
+
+void DataTree::removeLastDataLeaf() {
+ auto deletePosOrNull = algorithms::GetLowestRightBorderNodeWithMoreThanOneChildOrNull(_nodeStore, _rootNode.get());
+ ASSERT(deletePosOrNull.get() != nullptr, "Tree has only one leaf, can't shrink it.");
+
+ deleteLastChildSubtree(deletePosOrNull.get());
+
+ ifRootHasOnlyOneChildReplaceRootWithItsChild();
+}
+
+void DataTree::ifRootHasOnlyOneChildReplaceRootWithItsChild() {
+ DataInnerNode *rootNode = dynamic_cast(_rootNode.get());
+ ASSERT(rootNode != nullptr, "RootNode is not an inner node");
+ if (rootNode->numChildren() == 1) {
+ auto child = _nodeStore->load(rootNode->getChild(0)->key());
+ ASSERT(child != none, "Couldn't load first child of root node");
+ _rootNode = _nodeStore->overwriteNodeWith(std::move(_rootNode), **child);
+ _nodeStore->remove(std::move(*child));
+ }
+}
+
+void DataTree::deleteLastChildSubtree(DataInnerNode *node) {
+ auto lastChild = _nodeStore->load(node->LastChild()->key());
+ ASSERT(lastChild != none, "Couldn't load last child");
+ _nodeStore->removeSubtree(std::move(*lastChild));
+ node->removeLastChild();
+}
+
+unique_ref DataTree::addDataLeaf() {
+ auto insertPosOrNull = algorithms::GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(_nodeStore, _rootNode.get());
+ if (insertPosOrNull) {
+ return addDataLeafAt(insertPosOrNull.get());
+ } else {
+ return addDataLeafToFullTree();
+ }
+}
+
+unique_ref DataTree::addDataLeafAt(DataInnerNode *insertPos) {
+ auto new_leaf = _nodeStore->createNewLeafNode();
+ auto chain = createChainOfInnerNodes(insertPos->depth()-1, new_leaf.get());
+ insertPos->addChild(*chain);
+ return new_leaf;
+}
+
+optional_ownership_ptr DataTree::createChainOfInnerNodes(unsigned int num, DataNode *child) {
+ //TODO This function is implemented twice, once with optional_ownership_ptr, once with unique_ref. Redundancy!
+ optional_ownership_ptr chain = cpputils::WithoutOwnership(child);
+ for(unsigned int i=0; icreateNewInnerNode(*chain);
+ chain = cpputils::WithOwnership(std::move(newnode));
+ }
+ return chain;
+}
+
+unique_ref DataTree::createChainOfInnerNodes(unsigned int num, unique_ref child) {
+ unique_ref chain = std::move(child);
+ for(unsigned int i=0; icreateNewInnerNode(*chain);
+ }
+ return chain;
+}
+
+DataInnerNode* DataTree::increaseTreeDepth(unsigned int levels) {
+ ASSERT(levels >= 1, "Parameter out of bounds: tried to increase tree depth by zero.");
+ auto copyOfOldRoot = _nodeStore->createNewNodeAsCopyFrom(*_rootNode);
+ auto chain = createChainOfInnerNodes(levels-1, copyOfOldRoot.get());
+ auto newRootNode = DataNode::convertToNewInnerNode(std::move(_rootNode), *chain);
+ DataInnerNode *result = newRootNode.get();
+ _rootNode = std::move(newRootNode);
+ return result;
+}
+
+unique_ref DataTree::addDataLeafToFullTree() {
+ DataInnerNode *rootNode = increaseTreeDepth(1);
+ auto newLeaf = addDataLeafAt(rootNode);
+ return newLeaf;
+}
+
+const Key &DataTree::key() const {
+ return _rootNode->key();
+}
+
+void DataTree::flush() const {
+ // By grabbing a lock, we ensure that all modifying functions don't run currently and are therefore flushed
+ unique_lock lock(_mutex);
+ // We also have to flush the root node
+ _rootNode->flush();
+}
+
+unique_ref DataTree::releaseRootNode() {
+ return std::move(_rootNode);
+}
+
+//TODO Test numLeaves(), for example also two configurations with same number of bytes but different number of leaves (last leaf has 0 bytes)
+uint32_t DataTree::numLeaves() const {
+ shared_lock lock(_mutex);
+ return _numLeaves(*_rootNode);
+}
+
+uint32_t DataTree::_numLeaves(const DataNode &node) const {
+ const DataLeafNode *leaf = dynamic_cast(&node);
+ if (leaf != nullptr) {
+ return 1;
+ }
+
+ const DataInnerNode &inner = dynamic_cast(node);
+ uint64_t numLeavesInLeftChildren = (inner.numChildren()-1) * leavesPerFullChild(inner);
+ auto lastChild = _nodeStore->load(inner.LastChild()->key());
+ ASSERT(lastChild != none, "Couldn't load last child");
+ uint64_t numLeavesInRightChild = _numLeaves(**lastChild);
+
+ return numLeavesInLeftChildren + numLeavesInRightChild;
+}
+
+void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function func) {
+ //TODO Can we traverse in parallel?
+ unique_lock lock(_mutex); //TODO Only lock when resizing. Otherwise parallel read/write to a blob is not possible!
+ ASSERT(beginIndex <= endIndex, "Invalid parameters");
+ if (0 == endIndex) {
+ // In this case the utils::ceilLog(_, endIndex) below would fail
+ return;
+ }
+
+ uint8_t neededTreeDepth = utils::ceilLog(_nodeStore->layout().maxChildrenPerInnerNode(), endIndex);
+ uint32_t numLeaves = this->_numLeaves(*_rootNode); // TODO Querying the size causes a tree traversal down to the leaves. Possible without querying the size?
+ if (_rootNode->depth() < neededTreeDepth) {
+ //TODO Test cases that actually increase it here by 0 level / 1 level / more than 1 level
+ increaseTreeDepth(neededTreeDepth - _rootNode->depth());
+ }
+
+ if (numLeaves <= beginIndex) {
+ //TODO Test cases with numLeaves < / >= beginIndex
+ // There is a gap between the current size and the begin of the traversal
+ return _traverseLeaves(_rootNode.get(), 0, numLeaves-1, endIndex, [beginIndex, numLeaves, &func, this](DataLeafNode* node, uint32_t index) {
+ if (index >= beginIndex) {
+ func(node, index);
+ } else if (index == numLeaves - 1) {
+ // It is the old last leaf - resize it to maximum
+ node->resize(_nodeStore->layout().maxBytesPerLeaf());
+ }
+ });
+ } else if (numLeaves < endIndex) {
+ // We are starting traversal in the valid region, but traverse until after it (we grow new leaves)
+ return _traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, [numLeaves, &func, this] (DataLeafNode *node, uint32_t index) {
+ if (index == numLeaves - 1) {
+ // It is the old last leaf - resize it to maximum
+ node->resize(_nodeStore->layout().maxBytesPerLeaf());
+ }
+ func(node, index);
+ });
+ } else {
+ //We are traversing entirely inside the valid region
+ _traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, func);
+ }
+}
+
+void DataTree::_traverseLeaves(DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, function func) {
+ DataLeafNode *leaf = dynamic_cast(root);
+ if (leaf != nullptr) {
+ ASSERT(beginIndex <= 1 && endIndex <= 1, "If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1.");
+ if (beginIndex == 0 && endIndex == 1) {
+ func(leaf, leafOffset);
+ }
+ return;
+ }
+
+ DataInnerNode *inner = dynamic_cast(root);
+ uint32_t leavesPerChild = leavesPerFullChild(*inner);
+ uint32_t beginChild = beginIndex/leavesPerChild;
+ uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild);
+ vector> children = getOrCreateChildren(inner, beginChild, endChild);
+
+ for (uint32_t childIndex = beginChild; childIndex < endChild; ++childIndex) {
+ uint32_t childOffset = childIndex * leavesPerChild;
+ uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset);
+ uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
+ auto child = std::move(children[childIndex-beginChild]);
+ _traverseLeaves(child.get(), leafOffset + childOffset, localBeginIndex, localEndIndex, func);
+ }
+}
+
+vector> DataTree::getOrCreateChildren(DataInnerNode *node, uint32_t begin, uint32_t end) {
+ vector> children;
+ children.reserve(end-begin);
+ for (uint32_t childIndex = begin; childIndex < std::min(node->numChildren(), end); ++childIndex) {
+ auto child = _nodeStore->load(node->getChild(childIndex)->key());
+ ASSERT(child != none, "Couldn't load child node");
+ children.emplace_back(std::move(*child));
+ }
+ for (uint32_t childIndex = node->numChildren(); childIndex < end; ++childIndex) {
+ //TODO This creates each child with one chain to one leaf only, and then on the next lower level it
+ // has to create the children for the child. Would be faster to directly create full trees if necessary.
+ children.emplace_back(addChildTo(node));
+ }
+ ASSERT(children.size() == end-begin, "Number of children in the result is wrong");
+ return children;
+}
+
+unique_ref DataTree::addChildTo(DataInnerNode *node) {
+ auto new_leaf = _nodeStore->createNewLeafNode();
+ new_leaf->resize(_nodeStore->layout().maxBytesPerLeaf());
+ auto chain = createChainOfInnerNodes(node->depth()-1, std::move(new_leaf));
+ node->addChild(*chain);
+ return std::move(chain);
+}
+
+uint32_t DataTree::leavesPerFullChild(const DataInnerNode &root) const {
+ return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), (uint32_t)root.depth()-1);
+}
+
+uint64_t DataTree::numStoredBytes() const {
+ shared_lock lock(_mutex);
+ return _numStoredBytes();
+}
+
+uint64_t DataTree::_numStoredBytes() const {
+ return _numStoredBytes(*_rootNode);
+}
+
+uint64_t DataTree::_numStoredBytes(const DataNode &root) const {
+ const DataLeafNode *leaf = dynamic_cast(&root);
+ if (leaf != nullptr) {
+ return leaf->numBytes();
+ }
+
+ const DataInnerNode &inner = dynamic_cast(root);
+ uint64_t numBytesInLeftChildren = (inner.numChildren()-1) * leavesPerFullChild(inner) * _nodeStore->layout().maxBytesPerLeaf();
+ auto lastChild = _nodeStore->load(inner.LastChild()->key());
+ ASSERT(lastChild != none, "Couldn't load last child");
+ uint64_t numBytesInRightChild = _numStoredBytes(**lastChild);
+
+ return numBytesInLeftChildren + numBytesInRightChild;
+}
+
+void DataTree::resizeNumBytes(uint64_t newNumBytes) {
+ //TODO Can we resize in parallel? Especially creating new blocks (i.e. encrypting them) is expensive and should be done in parallel.
+ boost::upgrade_lock lock(_mutex);
+ {
+ boost::upgrade_to_unique_lock exclusiveLock(lock);
+ //TODO Faster implementation possible (no addDataLeaf()/removeLastDataLeaf() in a loop, but directly resizing)
+ LastLeaf(_rootNode.get())->resize(_nodeStore->layout().maxBytesPerLeaf());
+ uint64_t currentNumBytes = _numStoredBytes();
+ ASSERT(currentNumBytes % _nodeStore->layout().maxBytesPerLeaf() == 0, "The last leaf is not a max data leaf, although we just resized it to be one.");
+ uint32_t currentNumLeaves = currentNumBytes / _nodeStore->layout().maxBytesPerLeaf();
+ uint32_t newNumLeaves = std::max(UINT64_C(1), utils::ceilDivision(newNumBytes, _nodeStore->layout().maxBytesPerLeaf()));
+
+ for(uint32_t i = currentNumLeaves; i < newNumLeaves; ++i) {
+ addDataLeaf()->resize(_nodeStore->layout().maxBytesPerLeaf());
+ }
+ for(uint32_t i = currentNumLeaves; i > newNumLeaves; --i) {
+ removeLastDataLeaf();
+ }
+ uint32_t newLastLeafSize = newNumBytes - (newNumLeaves-1)*_nodeStore->layout().maxBytesPerLeaf();
+ LastLeaf(_rootNode.get())->resize(newLastLeafSize);
+ }
+ ASSERT(newNumBytes == _numStoredBytes(), "We resized to the wrong number of bytes ("+std::to_string(numStoredBytes())+" instead of "+std::to_string(newNumBytes)+")");
+}
+
+optional_ownership_ptr DataTree::LastLeaf(DataNode *root) {
+ DataLeafNode *leaf = dynamic_cast(root);
+ if (leaf != nullptr) {
+ return WithoutOwnership(leaf);
+ }
+
+ DataInnerNode *inner = dynamic_cast(root);
+ auto lastChild = _nodeStore->load(inner->LastChild()->key());
+ ASSERT(lastChild != none, "Couldn't load last child");
+ return WithOwnership(LastLeaf(std::move(*lastChild)));
+}
+
+unique_ref DataTree::LastLeaf(unique_ref root) {
+ auto leaf = dynamic_pointer_move(root);
+ if (leaf != none) {
+ return std::move(*leaf);
+ }
+ auto inner = dynamic_pointer_move(root);
+ ASSERT(inner != none, "Root node is neither a leaf nor an inner node");
+ auto child = _nodeStore->load((*inner)->LastChild()->key());
+ ASSERT(child != none, "Couldn't load last child");
+ return LastLeaf(std::move(*child));
+}
+
+uint64_t DataTree::maxBytesPerLeaf() const {
+ return _nodeStore->layout().maxBytesPerLeaf();
+}
+
+}
+}
+}
diff --git a/src/blobstore/implementations/onblocks/datatreestore/DataTree.h b/src/blobstore/implementations/onblocks/datatreestore/DataTree.h
new file mode 100644
index 00000000..ae7e6c43
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datatreestore/DataTree.h
@@ -0,0 +1,79 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREE_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREE_H_
+
+#include
+#include
+#include
+#include "../datanodestore/DataNodeView.h"
+//TODO Replace with C++14 once std::shared_mutex is supported
+#include
+#include
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+class DataNodeStore;
+class DataInnerNode;
+class DataLeafNode;
+class DataNode;
+}
+namespace datatreestore {
+
+//TODO It is strange that DataLeafNode is still part in the public interface of DataTree. This should be separated somehow.
+class DataTree final {
+public:
+ DataTree(datanodestore::DataNodeStore *nodeStore, cpputils::unique_ref rootNode);
+ ~DataTree();
+
+ const blockstore::Key &key() const;
+ //Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
+ uint64_t maxBytesPerLeaf() const;
+
+ void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function func);
+ void resizeNumBytes(uint64_t newNumBytes);
+
+ uint32_t numLeaves() const;
+ uint64_t numStoredBytes() const;
+
+ void flush() const;
+
+private:
+ mutable boost::shared_mutex _mutex;
+ datanodestore::DataNodeStore *_nodeStore;
+ cpputils::unique_ref _rootNode;
+
+ cpputils::unique_ref addDataLeaf();
+ void removeLastDataLeaf();
+
+ cpputils::unique_ref releaseRootNode();
+ friend class DataTreeStore;
+
+ cpputils::unique_ref addDataLeafAt(datanodestore::DataInnerNode *insertPos);
+ cpputils::optional_ownership_ptr createChainOfInnerNodes(unsigned int num, datanodestore::DataNode *child);
+ cpputils::unique_ref createChainOfInnerNodes(unsigned int num, cpputils::unique_ref child);
+ cpputils::unique_ref addDataLeafToFullTree();
+
+ void deleteLastChildSubtree(datanodestore::DataInnerNode *node);
+ void ifRootHasOnlyOneChildReplaceRootWithItsChild();
+
+ //TODO Use underscore for private methods
+ void _traverseLeaves(datanodestore::DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, std::function func);
+ uint32_t leavesPerFullChild(const datanodestore::DataInnerNode &root) const;
+ uint64_t _numStoredBytes() const;
+ uint64_t _numStoredBytes(const datanodestore::DataNode &root) const;
+ uint32_t _numLeaves(const datanodestore::DataNode &node) const;
+ cpputils::optional_ownership_ptr LastLeaf(datanodestore::DataNode *root);
+ cpputils::unique_ref LastLeaf(cpputils::unique_ref root);
+ datanodestore::DataInnerNode* increaseTreeDepth(unsigned int levels);
+ std::vector> getOrCreateChildren(datanodestore::DataInnerNode *node, uint32_t begin, uint32_t end);
+ cpputils::unique_ref addChildTo(datanodestore::DataInnerNode *node);
+
+ DISALLOW_COPY_AND_ASSIGN(DataTree);
+};
+
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.cpp b/src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.cpp
new file mode 100644
index 00000000..29fddbfd
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.cpp
@@ -0,0 +1,46 @@
+#include "DataTreeStore.h"
+#include "../datanodestore/DataNodeStore.h"
+#include "../datanodestore/DataLeafNode.h"
+#include "DataTree.h"
+
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using boost::optional;
+using boost::none;
+
+using blobstore::onblocks::datanodestore::DataNodeStore;
+using blobstore::onblocks::datanodestore::DataNode;
+
+namespace blobstore {
+namespace onblocks {
+namespace datatreestore {
+
+DataTreeStore::DataTreeStore(unique_ref nodeStore)
+ : _nodeStore(std::move(nodeStore)) {
+}
+
+DataTreeStore::~DataTreeStore() {
+}
+
+optional> DataTreeStore::load(const blockstore::Key &key) {
+ auto node = _nodeStore->load(key);
+ if (node == none) {
+ return none;
+ }
+ return make_unique_ref(_nodeStore.get(), std::move(*node));
+}
+
+unique_ref DataTreeStore::createNewTree() {
+ auto newleaf = _nodeStore->createNewLeafNode();
+ return make_unique_ref(_nodeStore.get(), std::move(newleaf));
+}
+
+void DataTreeStore::remove(unique_ref tree) {
+ auto root = tree->releaseRootNode();
+ cpputils::destruct(std::move(tree)); // Destruct tree
+ _nodeStore->removeSubtree(std::move(root));
+}
+
+}
+}
+}
diff --git a/src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h b/src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h
new file mode 100644
index 00000000..1697eab8
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h
@@ -0,0 +1,40 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_DATATREESTORE_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_DATATREESTORE_H_
+
+#include
+#include
+#include
+#include
+#include
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore {
+class DataNodeStore;
+}
+namespace datatreestore {
+class DataTree;
+
+class DataTreeStore final {
+public:
+ DataTreeStore(cpputils::unique_ref nodeStore);
+ ~DataTreeStore();
+
+ boost::optional> load(const blockstore::Key &key);
+
+ cpputils::unique_ref createNewTree();
+
+ void remove(cpputils::unique_ref tree);
+
+private:
+ cpputils::unique_ref _nodeStore;
+
+ DISALLOW_COPY_AND_ASSIGN(DataTreeStore);
+};
+
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.cpp b/src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.cpp
new file mode 100644
index 00000000..6f232700
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.cpp
@@ -0,0 +1,67 @@
+#include "algorithms.h"
+#include
+#include
+
+#include "../../datanodestore/DataInnerNode.h"
+#include "../../datanodestore/DataNodeStore.h"
+#include
+
+using std::function;
+using cpputils::optional_ownership_ptr;
+using cpputils::dynamic_pointer_move;
+using cpputils::unique_ref;
+using blobstore::onblocks::datanodestore::DataInnerNode;
+using blobstore::onblocks::datanodestore::DataNode;
+using blobstore::onblocks::datanodestore::DataNodeStore;
+using blockstore::Key;
+using boost::optional;
+using boost::none;
+
+namespace blobstore {
+namespace onblocks {
+namespace datatreestore {
+namespace algorithms {
+
+optional> getLastChildAsInnerNode(DataNodeStore *nodeStore, const DataInnerNode &node) {
+ Key key = node.LastChild()->key();
+ auto lastChild = nodeStore->load(key);
+ ASSERT(lastChild != none, "Couldn't load last child");
+ return dynamic_pointer_move(*lastChild);
+}
+
+//Returns the lowest right border node meeting the condition specified (exclusive the leaf).
+//Returns nullptr, if no inner right border node meets the condition.
+optional_ownership_ptr GetLowestInnerRightBorderNodeWithConditionOrNull(DataNodeStore *nodeStore, datanodestore::DataNode *rootNode, function condition) {
+ optional_ownership_ptr currentNode = cpputils::WithoutOwnership(dynamic_cast(rootNode));
+ optional_ownership_ptr result = cpputils::null();
+ for (unsigned int i=0; i < rootNode->depth(); ++i) {
+ //TODO This unnecessarily loads the leaf node in the last loop run
+ auto lastChild = getLastChildAsInnerNode(nodeStore, *currentNode);
+ if (condition(*currentNode)) {
+ result = std::move(currentNode);
+ }
+ ASSERT(lastChild != none || static_cast(i) == rootNode->depth()-1, "Couldn't get last child as inner node but we're not deep enough yet for the last child to be a leaf");
+ if (lastChild != none) {
+ currentNode = cpputils::WithOwnership(std::move(*lastChild));
+ }
+ }
+
+ return result;
+}
+
+optional_ownership_ptr GetLowestRightBorderNodeWithMoreThanOneChildOrNull(DataNodeStore *nodeStore, DataNode *rootNode) {
+ return GetLowestInnerRightBorderNodeWithConditionOrNull(nodeStore, rootNode, [] (const datanodestore::DataInnerNode &node) {
+ return node.numChildren() > 1;
+ });
+}
+
+optional_ownership_ptr GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode) {
+ return GetLowestInnerRightBorderNodeWithConditionOrNull(nodeStore, rootNode, [] (const datanodestore::DataInnerNode &node) {
+ return node.numChildren() < node.maxStoreableChildren();
+ });
+}
+
+}
+}
+}
+}
diff --git a/src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.h b/src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.h
new file mode 100644
index 00000000..a04168a8
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.h
@@ -0,0 +1,30 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_IMPL_ALGORITHMS_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_IMPL_ALGORITHMS_H_
+
+#include
+
+namespace blobstore {
+namespace onblocks {
+namespace datanodestore{
+class DataNode;
+class DataInnerNode;
+class DataNodeStore;
+}
+namespace datatreestore {
+namespace algorithms {
+
+//Returns the lowest right border node with at least two children.
+//Returns nullptr, if all right border nodes have only one child (since the root is a right border node, this means that the whole tree has exactly one leaf)
+cpputils::optional_ownership_ptr GetLowestRightBorderNodeWithMoreThanOneChildOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode);
+
+//Returns the lowest right border node with less than k children (not considering leaves).
+//Returns nullptr, if all right border nodes have k children (the tree is full)
+cpputils::optional_ownership_ptr GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode);
+
+}
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.cpp b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.cpp
new file mode 100644
index 00000000..8de9dedf
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.cpp
@@ -0,0 +1 @@
+#include "DataTreeRef.h"
diff --git a/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h
new file mode 100644
index 00000000..b379c91a
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h
@@ -0,0 +1,55 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_DATATREEREF_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_DATATREEREF_H_
+
+#include
+#include "../datatreestore/DataTree.h"
+
+namespace blobstore {
+namespace onblocks {
+namespace parallelaccessdatatreestore {
+
+class DataTreeRef final: public parallelaccessstore::ParallelAccessStore::ResourceRefBase {
+public:
+ DataTreeRef(datatreestore::DataTree *baseTree): _baseTree(baseTree) {}
+
+ const blockstore::Key &key() const {
+ return _baseTree->key();
+ }
+
+ uint64_t maxBytesPerLeaf() const {
+ return _baseTree->maxBytesPerLeaf();
+ }
+
+ void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function func) {
+ return _baseTree->traverseLeaves(beginIndex, endIndex, func);
+ }
+
+ uint32_t numLeaves() const {
+ return _baseTree->numLeaves();
+ }
+
+ void resizeNumBytes(uint64_t newNumBytes) {
+ return _baseTree->resizeNumBytes(newNumBytes);
+ }
+
+ uint64_t numStoredBytes() const {
+ return _baseTree->numStoredBytes();
+ }
+
+ void flush() {
+ return _baseTree->flush();
+ }
+
+private:
+
+ datatreestore::DataTree *_baseTree;
+
+ DISALLOW_COPY_AND_ASSIGN(DataTreeRef);
+};
+
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp
new file mode 100644
index 00000000..3061e359
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp
@@ -0,0 +1,44 @@
+#include "DataTreeRef.h"
+#include "ParallelAccessDataTreeStore.h"
+#include "ParallelAccessDataTreeStoreAdapter.h"
+#include "../datanodestore/DataNodeStore.h"
+#include "../datanodestore/DataLeafNode.h"
+
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using boost::optional;
+
+using blobstore::onblocks::datatreestore::DataTreeStore;
+using blockstore::Key;
+
+namespace blobstore {
+namespace onblocks {
+using datatreestore::DataTreeStore;
+using datatreestore::DataTree;
+namespace parallelaccessdatatreestore {
+
+ParallelAccessDataTreeStore::ParallelAccessDataTreeStore(unique_ref dataTreeStore)
+ : _dataTreeStore(std::move(dataTreeStore)), _parallelAccessStore(make_unique_ref(_dataTreeStore.get())) {
+}
+
+ParallelAccessDataTreeStore::~ParallelAccessDataTreeStore() {
+}
+
+optional> ParallelAccessDataTreeStore::load(const blockstore::Key &key) {
+ return _parallelAccessStore.load(key);
+}
+
+unique_ref ParallelAccessDataTreeStore::createNewTree() {
+ auto dataTree = _dataTreeStore->createNewTree();
+ Key key = dataTree->key();
+ return _parallelAccessStore.add(key, std::move(dataTree));
+}
+
+void ParallelAccessDataTreeStore::remove(unique_ref tree) {
+ Key key = tree->key();
+ return _parallelAccessStore.remove(key, std::move(tree));
+}
+
+}
+}
+}
diff --git a/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h
new file mode 100644
index 00000000..c0499a43
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h
@@ -0,0 +1,43 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTORE_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTORE_H_
+
+#include
+#include
+#include
+#include
+
+namespace blobstore {
+namespace onblocks {
+namespace datatreestore {
+class DataTreeStore;
+class DataTree;
+}
+namespace parallelaccessdatatreestore {
+class DataTreeRef;
+
+//TODO Test CachingDataTreeStore
+
+class ParallelAccessDataTreeStore final {
+public:
+ ParallelAccessDataTreeStore(cpputils::unique_ref dataTreeStore);
+ ~ParallelAccessDataTreeStore();
+
+ boost::optional> load(const blockstore::Key &key);
+
+ cpputils::unique_ref createNewTree();
+
+ void remove(cpputils::unique_ref tree);
+
+private:
+ cpputils::unique_ref _dataTreeStore;
+ parallelaccessstore::ParallelAccessStore _parallelAccessStore;
+
+ DISALLOW_COPY_AND_ASSIGN(ParallelAccessDataTreeStore);
+};
+
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.cpp b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.cpp
new file mode 100644
index 00000000..52ef9fc4
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.cpp
@@ -0,0 +1 @@
+#include "ParallelAccessDataTreeStoreAdapter.h"
diff --git a/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.h b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.h
new file mode 100644
index 00000000..6e8b1861
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.h
@@ -0,0 +1,38 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTOREADAPTER_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTOREADAPTER_H_
+
+#include
+#include
+#include "../datatreestore/DataTreeStore.h"
+#include "../datatreestore/DataTree.h"
+
+namespace blobstore {
+namespace onblocks {
+namespace parallelaccessdatatreestore {
+
+class ParallelAccessDataTreeStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore {
+public:
+ ParallelAccessDataTreeStoreAdapter(datatreestore::DataTreeStore *baseDataTreeStore)
+ :_baseDataTreeStore(std::move(baseDataTreeStore)) {
+ }
+
+ boost::optional> loadFromBaseStore(const blockstore::Key &key) override {
+ return _baseDataTreeStore->load(key);
+ }
+
+ void removeFromBaseStore(cpputils::unique_ref dataTree) override {
+ return _baseDataTreeStore->remove(std::move(dataTree));
+ }
+
+private:
+ datatreestore::DataTreeStore *_baseDataTreeStore;
+
+ DISALLOW_COPY_AND_ASSIGN(ParallelAccessDataTreeStoreAdapter);
+};
+
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/implementations/onblocks/utils/Math.cpp b/src/blobstore/implementations/onblocks/utils/Math.cpp
new file mode 100644
index 00000000..b00989a1
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/utils/Math.cpp
@@ -0,0 +1 @@
+#include "Math.h"
diff --git a/src/blobstore/implementations/onblocks/utils/Math.h b/src/blobstore/implementations/onblocks/utils/Math.h
new file mode 100644
index 00000000..370c1e01
--- /dev/null
+++ b/src/blobstore/implementations/onblocks/utils/Math.h
@@ -0,0 +1,43 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_UTILS_MATH_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_UTILS_MATH_H_
+
+#include
+#include
+
+namespace blobstore {
+namespace onblocks {
+namespace utils {
+
+template
+inline INT_TYPE intPow(INT_TYPE base, INT_TYPE exponent) {
+ INT_TYPE result = 1;
+ for(INT_TYPE i = 0; i < exponent; ++i) {
+ result *= base;
+ }
+ return result;
+}
+
+template
+inline INT_TYPE ceilDivision(INT_TYPE dividend, INT_TYPE divisor) {
+ return (dividend + divisor - 1)/divisor;
+}
+
+template
+inline INT_TYPE maxZeroSubtraction(INT_TYPE minuend, INT_TYPE subtrahend) {
+ if (minuend < subtrahend) {
+ return 0u;
+ }
+ return minuend-subtrahend;
+}
+
+template
+inline INT_TYPE ceilLog(INT_TYPE base, INT_TYPE value) {
+ return std::ceil((long double)std::log(value)/(long double)std::log(base));
+}
+
+}
+}
+}
+
+#endif
diff --git a/src/blobstore/interface/Blob.h b/src/blobstore/interface/Blob.h
new file mode 100644
index 00000000..972461dc
--- /dev/null
+++ b/src/blobstore/interface/Blob.h
@@ -0,0 +1,35 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_INTERFACE_BLOB_H_
+#define MESSMER_BLOBSTORE_INTERFACE_BLOB_H_
+
+#include
+#include
+#include
+#include
+
+namespace blobstore {
+
+class Blob {
+public:
+ virtual ~Blob() {}
+
+ //TODO Use own Key class for blobstore
+ virtual const blockstore::Key &key() const = 0;
+
+ virtual uint64_t size() const = 0;
+ virtual void resize(uint64_t numBytes) = 0;
+
+ virtual cpputils::Data readAll() const = 0;
+ virtual void read(void *target, uint64_t offset, uint64_t size) const = 0;
+ virtual uint64_t tryRead(void *target, uint64_t offset, uint64_t size) const = 0;
+ virtual void write(const void *source, uint64_t offset, uint64_t size) = 0;
+
+ virtual void flush() = 0;
+
+ //TODO Test tryRead
+};
+
+}
+
+
+#endif
diff --git a/src/blobstore/interface/BlobStore.h b/src/blobstore/interface/BlobStore.h
new file mode 100644
index 00000000..ad8b0822
--- /dev/null
+++ b/src/blobstore/interface/BlobStore.h
@@ -0,0 +1,25 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_INTERFACE_BLOBSTORE_H_
+#define MESSMER_BLOBSTORE_INTERFACE_BLOBSTORE_H_
+
+#include "Blob.h"
+#include
+#include
+
+#include
+#include
+
+namespace blobstore {
+
+class BlobStore {
+public:
+ virtual ~BlobStore() {}
+
+ virtual cpputils::unique_ref create() = 0;
+ virtual boost::optional> load(const blockstore::Key &key) = 0;
+ virtual void remove(cpputils::unique_ref blob) = 0;
+};
+
+}
+
+#endif
diff --git a/src/blockstore/CMakeLists.txt b/src/blockstore/CMakeLists.txt
new file mode 100644
index 00000000..e435b4cd
--- /dev/null
+++ b/src/blockstore/CMakeLists.txt
@@ -0,0 +1,43 @@
+project (blockstore)
+
+set(SOURCES
+ utils/Key.cpp
+ utils/BlockStoreUtils.cpp
+ utils/FileDoesntExistException.cpp
+ interface/helpers/BlockStoreWithRandomKeys.cpp
+ implementations/testfake/FakeBlockStore.cpp
+ implementations/testfake/FakeBlock.cpp
+ implementations/inmemory/InMemoryBlock.cpp
+ implementations/inmemory/InMemoryBlockStore.cpp
+ implementations/parallelaccess/ParallelAccessBlockStore.cpp
+ implementations/parallelaccess/BlockRef.cpp
+ implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp
+ implementations/compressing/CompressingBlockStore.cpp
+ implementations/compressing/CompressedBlock.cpp
+ implementations/compressing/compressors/RunLengthEncoding.cpp
+ implementations/compressing/compressors/Gzip.cpp
+ implementations/encrypted/EncryptedBlockStore.cpp
+ implementations/encrypted/EncryptedBlock.cpp
+ implementations/ondisk/OnDiskBlockStore.cpp
+ implementations/ondisk/OnDiskBlock.cpp
+ implementations/caching/CachingBlockStore.cpp
+ implementations/caching/cache/PeriodicTask.cpp
+ implementations/caching/cache/CacheEntry.cpp
+ implementations/caching/cache/Cache.cpp
+ implementations/caching/cache/QueueMap.cpp
+ implementations/caching/CachedBlock.cpp
+ implementations/caching/NewBlock.cpp
+)
+
+add_library(${PROJECT_NAME} STATIC ${SOURCES})
+
+# This is needed by boost thread
+if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+ target_link_libraries(${PROJECT_NAME} PRIVATE rt)
+endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+
+target_link_libraries(${PROJECT_NAME} PUBLIC cpp-utils)
+
+target_add_boost(${PROJECT_NAME} filesystem system thread)
+target_enable_style_warnings(${PROJECT_NAME})
+target_activate_cpp14(${PROJECT_NAME})
diff --git a/src/blockstore/implementations/caching/CachedBlock.cpp b/src/blockstore/implementations/caching/CachedBlock.cpp
new file mode 100644
index 00000000..8397888e
--- /dev/null
+++ b/src/blockstore/implementations/caching/CachedBlock.cpp
@@ -0,0 +1,46 @@
+#include "CachedBlock.h"
+#include "CachingBlockStore.h"
+
+using cpputils::unique_ref;
+
+namespace blockstore {
+namespace caching {
+
+CachedBlock::CachedBlock(unique_ref baseBlock, CachingBlockStore *blockStore)
+ :Block(baseBlock->key()),
+ _blockStore(blockStore),
+ _baseBlock(std::move(baseBlock)) {
+}
+
+CachedBlock::~CachedBlock() {
+ if (_baseBlock.get() != nullptr) {
+ _blockStore->release(std::move(_baseBlock));
+ }
+}
+
+const void *CachedBlock::data() const {
+ return _baseBlock->data();
+}
+
+void CachedBlock::write(const void *source, uint64_t offset, uint64_t size) {
+ return _baseBlock->write(source, offset, size);
+}
+
+void CachedBlock::flush() {
+ return _baseBlock->flush();
+}
+
+size_t CachedBlock::size() const {
+ return _baseBlock->size();
+}
+
+void CachedBlock::resize(size_t newSize) {
+ return _baseBlock->resize(newSize);
+}
+
+unique_ref CachedBlock::releaseBlock() {
+ return std::move(_baseBlock);
+}
+
+}
+}
diff --git a/src/blockstore/implementations/caching/CachedBlock.h b/src/blockstore/implementations/caching/CachedBlock.h
new file mode 100644
index 00000000..81f1b640
--- /dev/null
+++ b/src/blockstore/implementations/caching/CachedBlock.h
@@ -0,0 +1,39 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
+
+#include "../../interface/Block.h"
+
+#include
+
+namespace blockstore {
+namespace caching {
+class CachingBlockStore;
+
+class CachedBlock final: public Block {
+public:
+ //TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
+ CachedBlock(cpputils::unique_ref baseBlock, CachingBlockStore *blockStore);
+ ~CachedBlock();
+
+ const void *data() const override;
+ void write(const void *source, uint64_t offset, uint64_t size) override;
+ void flush() override;
+
+ size_t size() const override;
+
+ void resize(size_t newSize) override;
+
+ cpputils::unique_ref releaseBlock();
+
+private:
+ CachingBlockStore *_blockStore;
+ cpputils::unique_ref _baseBlock;
+
+ DISALLOW_COPY_AND_ASSIGN(CachedBlock);
+};
+
+}
+}
+
+#endif
diff --git a/src/blockstore/implementations/caching/CachingBlockStore.cpp b/src/blockstore/implementations/caching/CachingBlockStore.cpp
new file mode 100644
index 00000000..b9afd151
--- /dev/null
+++ b/src/blockstore/implementations/caching/CachingBlockStore.cpp
@@ -0,0 +1,92 @@
+#include "CachedBlock.h"
+#include "NewBlock.h"
+#include "CachingBlockStore.h"
+#include "../../interface/Block.h"
+
+#include