hardcoresushi/libcryfs
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Merge all git repositories into one

Browse Source
This commit is contained in:
Sebastian Messmer 2016-02-11 16:39:42 +01:00
commit c6e8052d93
989 changed files with 204181 additions and 393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
.gitignore vendored
View File

@ -1,35 +1,5 @@
# Compiled Object files
*.slo
*.lo
*.o
*.obj
# Precompiled Headers
*.gch
*.pch
# Compiled Dynamic libraries
*.so
*.dylib
*.dll
# Fortran module files
*.mod
# Compiled Static libraries
*.lai
*.la
*.a
*.lib
# Executables
*.exe
*.out
*.app
# Biicode directory
bii
bin
umltest.inner.sh
umltest.status
/build
/cmake
/.idea

6
.idea/vcs.xml generated Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

79
.travis.yml
View File

@ -1,17 +1,37 @@
language: cpp
sudo: required
dist: trusty
compiler:
- gcc
- clang
os:
- linux
- osx
matrix:
allow_failures:
- os: osx
- compiler: clang
addons:
apt:
packages:
- libcrypto++-dev
- libfuse-dev
install:
- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/update_gcc_version.sh
&& chmod +x update_gcc_version.sh
&& ./update_gcc_version.sh 4.8
&& rm update_gcc_version.sh
- sudo apt-get install libfuse-dev
# This is needed for packaging 7z distribution packages
- sudo apt-get install software-properties-common && sudo add-apt-repository ppa:george-edison55/precise-backports -y && sudo apt-get update
- sudo apt-get install cmake cmake-data rpm
# CryFS needs cmake >= 3.3, install it.
# Install boost
- wget -O boost.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.56.0/boost_1_56_0.tar.bz2/download
- tar -xf boost.tar.bz2
- cd boost_1_56_0
# TODO We should use clang as toolchain for building boost when clang is used for building our code
- ./bootstrap.sh --with-libraries=filesystem,thread,chrono
- sudo ./b2 -d0 install
- cd ..
- sudo rm -rf boost.tar.bz2 boost_1_56_0
# Install run_with_fuse.sh
- mkdir cmake
- cd cmake
- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/run_with_fuse.sh
- chmod +x run_with_fuse.sh
# Install cmake >= 3.3
- wget --no-check-certificate https://cmake.org/files/v3.3/cmake-3.3.2-Linux-x86_64.tar.gz
&& tar -xf cmake-3.3.2-Linux-x86_64.tar.gz
&& sudo cp -R cmake-3.3.2-Linux-x86_64/* /usr
@ -19,35 +39,14 @@ install:
- cmake --version
# Use /dev/urandom when /dev/random is accessed, because travis doesn't have enough entropy
- sudo cp -a /dev/urandom /dev/random
before_script:
- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/setup_biicode_project.sh
&& chmod +x setup_biicode_project.sh
&& ./setup_biicode_project.sh
&& rm setup_biicode_project.sh
script:
#The configure line is needed as a workaround for the following link, otherwise we wouldn't need "bii configure" at all because "bii build" calls it: http://forum.biicode.com/t/error-could-not-find-the-following-static-boost-libraries-boost-thread/374
- bii cpp:configure || bii cpp:configure
# Build cryfs executable
- bii cpp:build -- -j2
# Build and run test cases
- bii cpp:build --target messmer_cryfs_test_main -- -j2
- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/run_with_fuse.sh
&& chmod +x run_with_fuse.sh
&& ./run_with_fuse.sh "./bin/messmer_cryfs_test_main"
&& rm run_with_fuse.sh
# Make distribution packages
- bii clean
- bii cpp:configure -D CMAKE_BUILD_TYPE=Release
- bii build -- -j2
- cd bii/build/messmer_cryfs && make package && cd ../../..
after_success:
- bii user ${BII_USERNAME} -p ${BII_PASSWORD}
- bii publish
#deploy:
# provider: biicode
# user: ${BII_USERNAME}
# password:
# secure: ${BII_PASSWORD}
# on:
# branch: develop
- cmake ..
- make -j2
- make package -j2
- ./test/cpp-utils/cpp-utils-test
- ./run_with_fuse.sh ./test/fspp/fspp-test
- ./test/parallelaccessstore/parallelaccessstore-test
- ./test/blockstore/blockstore-test
- ./test/blobstore/blobstore-test
after_script:
- rm run_with_fuse.sh

110
CMakeLists.txt
View File

@ -1,26 +1,15 @@
# Earlier cmake versions generate .deb packages for which the package manager says they're bad quality
# and asks the user whether they really want to install it. Cmake 3.3 fixes this.
CMAKE_MINIMUM_REQUIRED(VERSION 3.3)
cmake_minimum_required(VERSION 3.3)
INCLUDE(messmer/cmake/tools)
INCLUDE(messmer/gitversion/cmake)
include(utils.cmake)
SETUP_GOOGLETEST()
require_gcc_version(4.8)
# Actually create targets: EXEcutables and libraries.
ADD_BII_TARGETS()
ACTIVATE_CPP14()
ADD_BOOST(program_options chrono)
ADD_DEFINITIONS(-D_FILE_OFFSET_BITS=64)
GIT_VERSION_INIT()
ENABLE_STYLE_WARNINGS()
SET_TARGET_PROPERTIES(${BII_src_main_TARGET} PROPERTIES OUTPUT_NAME cryfs)
add_subdirectory(vendor)
add_subdirectory(gitversion)
add_subdirectory(src)
add_subdirectory(test)
# Fix debfiles permissions. Unfortunately, git doesn't store file permissions.
# When installing the .deb package and these files have the wrong permissions, the package manager complains.
@ -64,88 +53,3 @@ SET(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/debfiles/pos
INCLUDE(CPack)
# You can safely delete lines from here...
###############################################################################
# REFERENCE #
###############################################################################
#
# This CMakeLists.txt file helps defining your block building and compiling
# To learn more about the CMake use with biicode, visit http://docs.biicode.com/c++.html
#
# ----------------------------------------------------
# NEW FEATURE! Include cmake files from remote blocks:
# -----------------------------------------------------
# Now you can handle cmake dependencies alike you do with c/c++:
#
# INCLUDE(user/block/myrecipe) # include myrecipe.cmake from remote user/block
#
# > EXAMPLE: Include our recipes and activate C++11 in your block (http://www.biicode.com/biicode/cmake)
#
# INCLUDE(biicode/cmake/tools) # Include tools.cmake file from "cmake" block from the "biicode" user
# ACTIVATE_CPP11(INTERFACE ${BII_BLOCK_TARGET})
#
# Remember to run "bii find" to download out cmake tools file
#
# ---------------------
# INIT_BIICODE_BLOCK()
# ---------------------
# This function creates several helper variables as ${BII_BLOCK_NAME} and ${BII_BLOCK_USER}
# Also it loads variables from the cmake/bii_user_block_vars.cmake
# ${BII_LIB_SRC} File list to create the library
# ${BII_LIB_TYPE} Empty (default, STATIC most casess) STATIC or SHARED
# ${BII_LIB_DEPS} Dependencies to other libraries (user2_block2, user3_blockX)
# ${BII_LIB_SYSTEM_HEADERS} System linking requirements as windows.h, pthread.h, etc
#
# You can use or modify them here, for example, to add or remove files from targets based on OS
# Or use typical cmake configurations done BEFORE defining targets. Examples:
# ADD_DEFINITIONS(-DFOO)
# FIND_PACKAGE(OpenGL QUIET)
# You can add INCLUDE_DIRECTORIES here too
#
# ---------------------
# ADD_BIICODE_TARGETS()
# ---------------------
#
# This function creates the following variables:
# ${BII_BLOCK_TARGET} Interface (no files) target for convenient configuration of all
# targets in this block, as the rest of targets always depend on it
# has name in the form "user_block_interface"
# ${BII_LIB_TARGET} Target library name, usually in the form "user_block". May not exist
# if BII_LIB_SRC is empty
# ${BII_BLOCK_TARGETS} List of all targets defined in this block
# ${BII_BLOCK_EXES} List of executables targets defined in this block
# ${BII_exe_name_TARGET}: Executable target (e.g. ${BII_main_TARGET}. You can also use
# directly the name of the executable target (e.g. user_block_main)
#
# > EXAMPLE: Add include directories to all targets of this block
#
# TARGET_INCLUDE_DIRECTORIES(${BII_BLOCK_TARGET} INTERFACE myincludedir)
#
# You can add private include directories to the Lib (if existing)
#
# > EXAMPLE: Link with pthread:
#
# TARGET_LINK_LIBRARIES(${BII_BLOCK_TARGET} INTERFACE pthread)
# or link against library:
# TARGET_LINK_LIBRARIES(${BII_LIB_TARGET} PUBLIC pthread)
# or directly use the library target name:
# TARGET_LINK_LIBRARIES(user_block PUBLIC pthread)
#
# NOTE: This can be also done adding pthread to ${BII_LIB_DEPS}
# BEFORE calling ADD_BIICODE_TARGETS()
#
# > EXAMPLE: how to activate C++11
#
# IF(APPLE)
# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11 -stdlib=libc++")
# ELSEIF (WIN32 OR UNIX)
# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11")
# ENDIF(APPLE)
#
# > EXAMPLE: Set properties to target
#
# SET_TARGET_PROPERTIES(${BII_BLOCK_TARGET} PROPERTIES COMPILE_DEFINITIONS "IOV_MAX=255")
#

165
CMakeLists.txt~ Normal file
View File

@ -0,0 +1,165 @@
<<<<<<< HEAD
# Earlier cmake versions generate .deb packages for which the package manager says they're bad quality
# and asks the user whether they really want to install it. Cmake 3.3 fixes this.
CMAKE_MINIMUM_REQUIRED(VERSION 3.3)
INCLUDE(messmer/cmake/tools)
INCLUDE(messmer/gitversion/cmake)
SETUP_GOOGLETEST()
# Actually create targets: EXEcutables and libraries.
ADD_BII_TARGETS()
ACTIVATE_CPP14()
ADD_BOOST(program_options chrono)
ADD_DEFINITIONS(-D_FILE_OFFSET_BITS=64)
GIT_VERSION_INIT()
ENABLE_STYLE_WARNINGS()
SET_TARGET_PROPERTIES(${BII_src_main_TARGET} PROPERTIES OUTPUT_NAME cryfs)
# Fix debfiles permissions. Unfortunately, git doesn't store file permissions.
# When installing the .deb package and these files have the wrong permissions, the package manager complains.
EXECUTE_PROCESS(COMMAND /bin/bash -c "chmod 0755 ${CMAKE_CURRENT_SOURCE_DIR}/debfiles/*")
INSTALL(TARGETS ${BII_src_main_TARGET}
DESTINATION bin
CONFIGURATIONS Release)
SET(CPACK_GENERATOR TGZ DEB RPM)
SET(CPACK_PACKAGE_NAME "cryfs")
SET(CPACK_PACKAGE_VERSION "${GITVERSION_VERSION_STRING}")
SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Encrypt your files and store them in the cloud.")
SET(CPACK_PACKAGE_DESCRIPTION "CryFS encrypts your files, so you can safely store them anywhere. It works well together with cloud services like Dropbox, iCloud, OneDrive and others.")
SET(CPACK_PACKAGE_CONTACT "Sebastian Messmer <messmer@cryfs.org>")
SET(CPACK_PACKAGE_VENDOR "Sebastian Messmer")
SET(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE")
SET(CPACK_PACKAGE_INSTALL_DIRECTORY "CMake ${CMake_VERSION_MAJOR}.${CMake_VERSION_MINOR}")
IF(WIN32 AND NOT UNIX)
# There is a bug in NSI that does not handle full unix paths properly. Make
# sure there is at least one set of four (4) backlasshes.
#SET(CPACK_PACKAGE_ICON "${CMake_SOURCE_DIR}/Utilities/Release\\\\InstallIcon.bmp")
#SET(CPACK_NSIS_INSTALLED_ICON_NAME "bin\\\\cryfs.exe")
#SET(CPACK_NSIS_DISPLAY_NAME "${CPACK_PACKAGE_INSTALL_DIRECTORY} CryFS")
#SET(CPACK_NSIS_HELP_LINK "http:\\\\\\\\www.cryfs.org")
#SET(CPACK_NSIS_URL_INFO_ABOUT "http:\\\\\\\\www.cryfs.org")
#SET(CPACK_NSIS_CONTACT "messmer@cryfs.org")
#SET(CPACK_NSIS_MODIFY_PATH ON)
ELSE(WIN32 AND NOT UNIX)
SET(CPACK_STRIP_FILES "bin/cryfs")
SET(CPACK_SOURCE_STRIP_FILES "")
ENDIF(WIN32 AND NOT UNIX)
SET(CPACK_PACKAGE_EXECUTABLES "cryfs" "CryFS")
SET(CPACK_DEBIAN_PACKAGE_SECTION "utils")
SET(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
SET(CPACK_DEBIAN_PACKAGE_RECOMMENDS "fuse")
SET(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://www.cryfs.org")
SET(CPACK_RPM_PACKAGE_LICENSE "LGPLv3")
SET(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/debfiles/postinst;${CMAKE_CURRENT_SOURCE_DIR}/debfiles/postrm")
INCLUDE(CPack)
# You can safely delete lines from here...
###############################################################################
# REFERENCE #
###############################################################################
#
# This CMakeLists.txt file helps defining your block building and compiling
# To learn more about the CMake use with biicode, visit http://docs.biicode.com/c++.html
#
# ----------------------------------------------------
# NEW FEATURE! Include cmake files from remote blocks:
# -----------------------------------------------------
# Now you can handle cmake dependencies alike you do with c/c++:
#
# INCLUDE(user/block/myrecipe) # include myrecipe.cmake from remote user/block
#
# > EXAMPLE: Include our recipes and activate C++11 in your block (http://www.biicode.com/biicode/cmake)
#
# INCLUDE(biicode/cmake/tools) # Include tools.cmake file from "cmake" block from the "biicode" user
# ACTIVATE_CPP11(INTERFACE ${BII_BLOCK_TARGET})
#
# Remember to run "bii find" to download out cmake tools file
#
# ---------------------
# INIT_BIICODE_BLOCK()
# ---------------------
# This function creates several helper variables as ${BII_BLOCK_NAME} and ${BII_BLOCK_USER}
# Also it loads variables from the cmake/bii_user_block_vars.cmake
# ${BII_LIB_SRC} File list to create the library
# ${BII_LIB_TYPE} Empty (default, STATIC most casess) STATIC or SHARED
# ${BII_LIB_DEPS} Dependencies to other libraries (user2_block2, user3_blockX)
# ${BII_LIB_SYSTEM_HEADERS} System linking requirements as windows.h, pthread.h, etc
#
# You can use or modify them here, for example, to add or remove files from targets based on OS
# Or use typical cmake configurations done BEFORE defining targets. Examples:
# ADD_DEFINITIONS(-DFOO)
# FIND_PACKAGE(OpenGL QUIET)
# You can add INCLUDE_DIRECTORIES here too
#
# ---------------------
# ADD_BIICODE_TARGETS()
# ---------------------
#
# This function creates the following variables:
# ${BII_BLOCK_TARGET} Interface (no files) target for convenient configuration of all
# targets in this block, as the rest of targets always depend on it
# has name in the form "user_block_interface"
# ${BII_LIB_TARGET} Target library name, usually in the form "user_block". May not exist
# if BII_LIB_SRC is empty
# ${BII_BLOCK_TARGETS} List of all targets defined in this block
# ${BII_BLOCK_EXES} List of executables targets defined in this block
# ${BII_exe_name_TARGET}: Executable target (e.g. ${BII_main_TARGET}. You can also use
# directly the name of the executable target (e.g. user_block_main)
#
# > EXAMPLE: Add include directories to all targets of this block
#
# TARGET_INCLUDE_DIRECTORIES(${BII_BLOCK_TARGET} INTERFACE myincludedir)
#
# You can add private include directories to the Lib (if existing)
#
# > EXAMPLE: Link with pthread:
#
# TARGET_LINK_LIBRARIES(${BII_BLOCK_TARGET} INTERFACE pthread)
# or link against library:
# TARGET_LINK_LIBRARIES(${BII_LIB_TARGET} PUBLIC pthread)
# or directly use the library target name:
# TARGET_LINK_LIBRARIES(user_block PUBLIC pthread)
#
# NOTE: This can be also done adding pthread to ${BII_LIB_DEPS}
# BEFORE calling ADD_BIICODE_TARGETS()
#
# > EXAMPLE: how to activate C++11
#
# IF(APPLE)
# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11 -stdlib=libc++")
# ELSEIF (WIN32 OR UNIX)
# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11")
# ENDIF(APPLE)
#
# > EXAMPLE: Set properties to target
#
# SET_TARGET_PROPERTIES(${BII_BLOCK_TARGET} PROPERTIES COMPILE_DEFINITIONS "IOV_MAX=255")
#
=======
# Earlier cmake versions generate .deb packages for which the package manager says they're bad quality
# and asks the user whether they really want to install it. Cmake 3.3 fixes this.
cmake_minimum_required(VERSION 3.3)
include(utils.cmake)
require_gcc_version(4.8)
add_subdirectory(vendor)
add_subdirectory(src)
add_subdirectory(test)
>>>>>>> cpu/cmake

2
README.md
View File

@ -51,7 +51,7 @@ Requirements
- GCC version >= 4.8 or Clang (TODO which minimal version?)
- CMake version >= 3.3
- libcurl4 (including development headers)
- Boost libraries filesystem, system, chrono, thread in version >= 1.56
- Boost libraries filesystem, system, chrono, program_options, thread in version >= 1.56
- Crypto++ >= 5.6.3 (TODO Lower minimal version possible?)
- libFUSE >= 2.8.6 (including development headers)

7
src/CMakeLists.txt Normal file
View File

@ -0,0 +1,7 @@
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
add_subdirectory(cpp-utils)
add_subdirectory(fspp)
add_subdirectory(blockstore)
add_subdirectory(blobstore)
add_subdirectory(cryfs)

29
src/blobstore/CMakeLists.txt Normal file
View File

@ -0,0 +1,29 @@
project (blobstore)
set(SOURCES
implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.cpp
implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.cpp
implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp
implementations/onblocks/utils/Math.cpp
implementations/onblocks/BlobStoreOnBlocks.cpp
implementations/onblocks/datanodestore/DataNode.cpp
implementations/onblocks/datanodestore/DataLeafNode.cpp
implementations/onblocks/datanodestore/DataInnerNode.cpp
implementations/onblocks/datanodestore/DataNodeStore.cpp
implementations/onblocks/datatreestore/impl/algorithms.cpp
implementations/onblocks/datatreestore/DataTree.cpp
implementations/onblocks/datatreestore/DataTreeStore.cpp
implementations/onblocks/BlobOnBlocks.cpp
)
add_library(${PROJECT_NAME} STATIC ${SOURCES})
# This is needed by boost thread
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PRIVATE rt)
endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PUBLIC cpp-utils blockstore)
target_add_boost(${PROJECT_NAME} filesystem system thread)
target_enable_style_warnings(${PROJECT_NAME})
target_activate_cpp14(${PROJECT_NAME})

109
src/blobstore/implementations/onblocks/BlobOnBlocks.cpp Normal file
View File

@ -0,0 +1,109 @@
#include "parallelaccessdatatreestore/DataTreeRef.h"
#include "BlobOnBlocks.h"
#include "datanodestore/DataLeafNode.h"
#include "utils/Math.h"
#include <cmath>
#include <cpp-utils/assert/assert.h>
using std::function;
using cpputils::unique_ref;
using cpputils::Data;
using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataNodeLayout;
using blockstore::Key;
namespace blobstore {
namespace onblocks {
using parallelaccessdatatreestore::DataTreeRef;
BlobOnBlocks::BlobOnBlocks(unique_ref<DataTreeRef> datatree)
: _datatree(std::move(datatree)), _sizeCache(boost::none) {
}
BlobOnBlocks::~BlobOnBlocks() {
}
uint64_t BlobOnBlocks::size() const {
if (_sizeCache == boost::none) {
_sizeCache = _datatree->numStoredBytes();
}
return *_sizeCache;
}
void BlobOnBlocks::resize(uint64_t numBytes) {
_datatree->resizeNumBytes(numBytes);
_sizeCache = numBytes;
}
void BlobOnBlocks::traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function<void (uint64_t, DataLeafNode *leaf, uint32_t, uint32_t)> func) const {
uint64_t endByte = beginByte + sizeBytes;
uint32_t firstLeaf = beginByte / _datatree->maxBytesPerLeaf();
uint32_t endLeaf = utils::ceilDivision(endByte, _datatree->maxBytesPerLeaf());
bool writingOutside = size() < endByte; // TODO Calling size() is slow because it has to traverse the tree
_datatree->traverseLeaves(firstLeaf, endLeaf, [&func, beginByte, endByte, endLeaf, writingOutside](DataLeafNode *leaf, uint32_t leafIndex) {
uint64_t indexOfFirstLeafByte = leafIndex * leaf->maxStoreableBytes();
uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
uint32_t dataEnd = std::min(leaf->maxStoreableBytes(), endByte - indexOfFirstLeafByte);
if (leafIndex == endLeaf-1 && writingOutside) {
// If we are traversing an area that didn't exist before, then the last leaf was just created with a wrong size. We have to fix it.
leaf->resize(dataEnd);
}
func(indexOfFirstLeafByte, leaf, dataBegin, dataEnd-dataBegin);
});
if (writingOutside) {
ASSERT(_datatree->numStoredBytes() == endByte, "Writing didn't grow by the correct number of bytes");
_sizeCache = endByte;
}
}
Data BlobOnBlocks::readAll() const {
//TODO Querying size is inefficient. Is this possible without a call to size()?
uint64_t count = size();
Data result(count);
_read(result.data(), 0, count);
return result;
}
void BlobOnBlocks::read(void *target, uint64_t offset, uint64_t count) const {
ASSERT(offset <= size() && offset + count <= size(), "BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
uint64_t read = tryRead(target, offset, count);
ASSERT(read == count, "BlobOnBlocks::read() couldn't read all requested bytes. Use BlobOnBlocks::tryRead() if this should be allowed.");
}
uint64_t BlobOnBlocks::tryRead(void *target, uint64_t offset, uint64_t count) const {
//TODO Quite inefficient to call size() here, because that has to traverse the tree
uint64_t realCount = std::max(UINT64_C(0), std::min(count, size()-offset));
_read(target, offset, realCount);
return realCount;
}
void BlobOnBlocks::_read(void *target, uint64_t offset, uint64_t count) const {
traverseLeaves(offset, count, [target, offset] (uint64_t indexOfFirstLeafByte, const DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
//TODO Simplify formula, make it easier to understand
leaf->read((uint8_t*)target + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
});
}
void BlobOnBlocks::write(const void *source, uint64_t offset, uint64_t count) {
traverseLeaves(offset, count, [source, offset] (uint64_t indexOfFirstLeafByte, DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
//TODO Simplify formula, make it easier to understand
leaf->write((uint8_t*)source + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
});
}
void BlobOnBlocks::flush() {
_datatree->flush();
}
const Key &BlobOnBlocks::key() const {
return _datatree->key();
}
unique_ref<DataTreeRef> BlobOnBlocks::releaseTree() {
return std::move(_datatree);
}
}
}

52
src/blobstore/implementations/onblocks/BlobOnBlocks.h Normal file
View File

@ -0,0 +1,52 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_BLOBONBLOCKS_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_BLOBONBLOCKS_H_
#include "../../interface/Blob.h"
#include <memory>
#include <boost/optional.hpp>
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataLeafNode;
}
namespace parallelaccessdatatreestore {
class DataTreeRef;
}
class BlobOnBlocks final: public Blob {
public:
BlobOnBlocks(cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> datatree);
~BlobOnBlocks();
const blockstore::Key &key() const override;
uint64_t size() const override;
void resize(uint64_t numBytes) override;
cpputils::Data readAll() const override;
void read(void *target, uint64_t offset, uint64_t size) const override;
uint64_t tryRead(void *target, uint64_t offset, uint64_t size) const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> releaseTree();
private:
void _read(void *target, uint64_t offset, uint64_t count) const;
void traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function<void (uint64_t, datanodestore::DataLeafNode *, uint32_t, uint32_t)>) const;
cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> _datatree;
mutable boost::optional<uint64_t> _sizeCache;
DISALLOW_COPY_AND_ASSIGN(BlobOnBlocks);
};
}
}
#endif

56
src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp Normal file
View File

@ -0,0 +1,56 @@
#include "parallelaccessdatatreestore/DataTreeRef.h"
#include "parallelaccessdatatreestore/ParallelAccessDataTreeStore.h"
#include <blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h>
#include "datanodestore/DataLeafNode.h"
#include "datanodestore/DataNodeStore.h"
#include "datatreestore/DataTreeStore.h"
#include "datatreestore/DataTree.h"
#include "BlobStoreOnBlocks.h"
#include "BlobOnBlocks.h"
#include <cpp-utils/pointer/cast.h>
#include <cpp-utils/assert/assert.h>
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using blockstore::BlockStore;
using blockstore::parallelaccess::ParallelAccessBlockStore;
using blockstore::Key;
using cpputils::dynamic_pointer_move;
using boost::optional;
using boost::none;
namespace blobstore {
namespace onblocks {
using datanodestore::DataNodeStore;
using datatreestore::DataTreeStore;
using parallelaccessdatatreestore::ParallelAccessDataTreeStore;
BlobStoreOnBlocks::BlobStoreOnBlocks(unique_ref<BlockStore> blockStore, uint32_t blocksizeBytes)
: _dataTreeStore(make_unique_ref<ParallelAccessDataTreeStore>(make_unique_ref<DataTreeStore>(make_unique_ref<DataNodeStore>(make_unique_ref<ParallelAccessBlockStore>(std::move(blockStore)), blocksizeBytes)))) {
}
BlobStoreOnBlocks::~BlobStoreOnBlocks() {
}
unique_ref<Blob> BlobStoreOnBlocks::create() {
return make_unique_ref<BlobOnBlocks>(_dataTreeStore->createNewTree());
}
optional<unique_ref<Blob>> BlobStoreOnBlocks::load(const Key &key) {
auto tree = _dataTreeStore->load(key);
if (tree == none) {
return none;
}
return optional<unique_ref<Blob>>(make_unique_ref<BlobOnBlocks>(std::move(*tree)));
}
void BlobStoreOnBlocks::remove(unique_ref<Blob> blob) {
auto _blob = dynamic_pointer_move<BlobOnBlocks>(blob);
ASSERT(_blob != none, "Passed Blob in BlobStoreOnBlocks::remove() is not a BlobOnBlocks.");
_dataTreeStore->remove((*_blob)->releaseTree());
}
}
}

35
src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h Normal file
View File

@ -0,0 +1,35 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_BLOCKED_BLOBSTOREONBLOCKS_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_BLOCKED_BLOBSTOREONBLOCKS_H_
#include "../../interface/BlobStore.h"
#include <blockstore/interface/BlockStore.h>
namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class ParallelAccessDataTreeStore;
}
//TODO Make blobstore able to cope with incomplete data (some blocks missing, because they're not synchronized yet) and write test cases for that
class BlobStoreOnBlocks final: public BlobStore {
public:
BlobStoreOnBlocks(cpputils::unique_ref<blockstore::BlockStore> blockStore, uint32_t blocksizeBytes);
~BlobStoreOnBlocks();
cpputils::unique_ref<Blob> create() override;
boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::Key &key) override;
void remove(cpputils::unique_ref<Blob> blob) override;
private:
cpputils::unique_ref<parallelaccessdatatreestore::ParallelAccessDataTreeStore> _dataTreeStore;
DISALLOW_COPY_AND_ASSIGN(BlobStoreOnBlocks);
};
}
}
#endif

87
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp Normal file
View File

@ -0,0 +1,87 @@
#include "DataInnerNode.h"
#include "DataNodeStore.h"
#include <cpp-utils/assert/assert.h>
using blockstore::Block;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using blockstore::Key;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
DataInnerNode::DataInnerNode(DataNodeView view)
: DataNode(std::move(view)) {
ASSERT(depth() > 0, "Inner node can't have depth 0. Is this a leaf maybe?");
}
DataInnerNode::~DataInnerNode() {
}
unique_ref<DataInnerNode> DataInnerNode::InitializeNewNode(unique_ref<Block> block, const DataNode &first_child) {
DataNodeView node(std::move(block));
node.setDepth(first_child.depth() + 1);
node.setSize(1);
auto result = make_unique_ref<DataInnerNode>(std::move(node));
result->ChildrenBegin()->setKey(first_child.key());
return result;
}
uint32_t DataInnerNode::numChildren() const {
return node().Size();
}
DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->ChildrenBegin());
}
const DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() const {
return node().DataBegin<ChildEntry>();
}
DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->ChildrenEnd());
}
const DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() const {
return ChildrenBegin() + node().Size();
}
DataInnerNode::ChildEntry *DataInnerNode::LastChild() {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->LastChild());
}
const DataInnerNode::ChildEntry *DataInnerNode::LastChild() const {
return getChild(numChildren()-1);
}
DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->getChild(index));
}
const DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) const {
ASSERT(index < numChildren(), "Accessing child out of range");
return ChildrenBegin()+index;
}
void DataInnerNode::addChild(const DataNode &child) {
ASSERT(numChildren() < maxStoreableChildren(), "Adding more children than we can store");
ASSERT(child.depth() == depth()-1, "The child that should be added has wrong depth");
node().setSize(node().Size()+1);
LastChild()->setKey(child.key());
}
void DataInnerNode::removeLastChild() {
ASSERT(node().Size() > 1, "There is no child to remove");
node().setSize(node().Size()-1);
}
uint32_t DataInnerNode::maxStoreableChildren() const {
return node().layout().maxChildrenPerInnerNode();
}
}
}
}

49
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.h Normal file
View File

@ -0,0 +1,49 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_H_
#include "DataNode.h"
#include "DataInnerNode_ChildEntry.h"
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataInnerNode final: public DataNode {
public:
static cpputils::unique_ref<DataInnerNode> InitializeNewNode(cpputils::unique_ref<blockstore::Block> block, const DataNode &first_child_key);
DataInnerNode(DataNodeView block);
~DataInnerNode();
using ChildEntry = DataInnerNode_ChildEntry;
uint32_t maxStoreableChildren() const;
ChildEntry *getChild(unsigned int index);
const ChildEntry *getChild(unsigned int index) const;
uint32_t numChildren() const;
void addChild(const DataNode &child_key);
void removeLastChild();
ChildEntry *LastChild();
const ChildEntry *LastChild() const;
private:
ChildEntry *ChildrenBegin();
ChildEntry *ChildrenEnd();
const ChildEntry *ChildrenBegin() const;
const ChildEntry *ChildrenEnd() const;
DISALLOW_COPY_AND_ASSIGN(DataInnerNode);
};
}
}
}
#endif

30
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode_ChildEntry.h Normal file
View File

@ -0,0 +1,30 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_CHILDENTRY_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_CHILDENTRY_H_
#include <cpp-utils/macros.h>
namespace blobstore{
namespace onblocks{
namespace datanodestore{
struct DataInnerNode_ChildEntry final {
public:
blockstore::Key key() const {
return blockstore::Key::FromBinary(_keydata);
}
private:
void setKey(const blockstore::Key &key) {
key.ToBinary(_keydata);
}
friend class DataInnerNode;
uint8_t _keydata[blockstore::Key::BINARY_LENGTH];
DISALLOW_COPY_AND_ASSIGN(DataInnerNode_ChildEntry);
};
}
}
}
#endif

67
src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.cpp Normal file
View File

@ -0,0 +1,67 @@
#include "DataLeafNode.h"
#include "DataInnerNode.h"
#include <cpp-utils/assert/assert.h>
using blockstore::Block;
using cpputils::Data;
using blockstore::Key;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
DataLeafNode::DataLeafNode(DataNodeView view)
: DataNode(std::move(view)) {
ASSERT(node().Depth() == 0, "Leaf node must have depth 0. Is it an inner node instead?");
ASSERT(numBytes() <= maxStoreableBytes(), "Leaf says it stores more bytes than it has space for");
}
DataLeafNode::~DataLeafNode() {
}
unique_ref<DataLeafNode> DataLeafNode::InitializeNewNode(unique_ref<Block> block) {
DataNodeView node(std::move(block));
node.setDepth(0);
node.setSize(0);
//fillDataWithZeroes(); not needed, because a newly created block will be zeroed out. DataLeafNodeTest.SpaceIsZeroFilledWhenGrowing ensures this.
return make_unique_ref<DataLeafNode>(std::move(node));
}
void DataLeafNode::read(void *target, uint64_t offset, uint64_t size) const {
ASSERT(offset <= node().Size() && offset + size <= node().Size(), "Read out of valid area"); // Also check offset, because the addition could lead to overflows
std::memcpy(target, (uint8_t*)node().data() + offset, size);
}
void DataLeafNode::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= node().Size() && offset + size <= node().Size(), "Write out of valid area"); // Also check offset, because the addition could lead to overflows
node().write(source, offset, size);
}
uint32_t DataLeafNode::numBytes() const {
return node().Size();
}
void DataLeafNode::resize(uint32_t new_size) {
ASSERT(new_size <= maxStoreableBytes(), "Trying to resize to a size larger than the maximal size");
uint32_t old_size = node().Size();
if (new_size < old_size) {
fillDataWithZeroesFromTo(new_size, old_size);
}
node().setSize(new_size);
}
void DataLeafNode::fillDataWithZeroesFromTo(off_t begin, off_t end) {
Data ZEROES(end-begin);
ZEROES.FillWithZeroes();
node().write(ZEROES.data(), begin, end-begin);
}
uint64_t DataLeafNode::maxStoreableBytes() const {
return node().layout().maxBytesPerLeaf();
}
}
}
}

39
src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.h Normal file
View File

@ -0,0 +1,39 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATALEAFNODE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATALEAFNODE_H_
#include "DataNode.h"
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataInnerNode;
class DataLeafNode final: public DataNode {
public:
static cpputils::unique_ref<DataLeafNode> InitializeNewNode(cpputils::unique_ref<blockstore::Block> block);
DataLeafNode(DataNodeView block);
~DataLeafNode();
//Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
uint64_t maxStoreableBytes() const;
void read(void *target, uint64_t offset, uint64_t size) const;
void write(const void *source, uint64_t offset, uint64_t size);
uint32_t numBytes() const;
void resize(uint32_t size);
private:
void fillDataWithZeroesFromTo(off_t begin, off_t end);
DISALLOW_COPY_AND_ASSIGN(DataLeafNode);
};
}
}
}
#endif

54
src/blobstore/implementations/onblocks/datanodestore/DataNode.cpp Normal file
View File

@ -0,0 +1,54 @@
#include "DataInnerNode.h"
#include "DataLeafNode.h"
#include "DataNode.h"
#include "DataNodeStore.h"
#include <blockstore/utils/BlockStoreUtils.h>
using blockstore::Block;
using blockstore::Key;
using std::runtime_error;
using cpputils::unique_ref;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
DataNode::DataNode(DataNodeView node)
: _node(std::move(node)) {
}
DataNode::~DataNode() {
}
DataNodeView &DataNode::node() {
return const_cast<DataNodeView&>(const_cast<const DataNode*>(this)->node());
}
const DataNodeView &DataNode::node() const {
return _node;
}
const Key &DataNode::key() const {
return _node.key();
}
uint8_t DataNode::depth() const {
return _node.Depth();
}
unique_ref<DataInnerNode> DataNode::convertToNewInnerNode(unique_ref<DataNode> node, const DataNode &first_child) {
Key key = node->key();
auto block = node->_node.releaseBlock();
blockstore::utils::fillWithZeroes(block.get());
return DataInnerNode::InitializeNewNode(std::move(block), first_child);
}
void DataNode::flush() const {
_node.flush();
}
}
}
}

44
src/blobstore/implementations/onblocks/datanodestore/DataNode.h Normal file
View File

@ -0,0 +1,44 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODE_H_
#include "DataNodeView.h"
#include <cpp-utils/data/Data.h>
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNodeStore;
class DataInnerNode;
class DataNode {
public:
virtual ~DataNode();
const blockstore::Key &key() const;
uint8_t depth() const;
static cpputils::unique_ref<DataInnerNode> convertToNewInnerNode(cpputils::unique_ref<DataNode> node, const DataNode &first_child);
void flush() const;
protected:
DataNode(DataNodeView block);
DataNodeView &node();
const DataNodeView &node() const;
friend class DataNodeStore;
private:
DataNodeView _node;
DISALLOW_COPY_AND_ASSIGN(DataNode);
};
}
}
}
#endif

113
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp Normal file
View File

@ -0,0 +1,113 @@
#include "DataInnerNode.h"
#include "DataLeafNode.h"
#include "DataNodeStore.h"
#include <blockstore/interface/BlockStore.h>
#include <blockstore/interface/Block.h>
#include <blockstore/utils/BlockStoreUtils.h>
#include <cpp-utils/assert/assert.h>
using blockstore::BlockStore;
using blockstore::Block;
using blockstore::Key;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using std::runtime_error;
using boost::optional;
using boost::none;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
DataNodeStore::DataNodeStore(unique_ref<BlockStore> blockstore, uint32_t blocksizeBytes)
: _blockstore(std::move(blockstore)), _layout(blocksizeBytes) {
}
DataNodeStore::~DataNodeStore() {
}
unique_ref<DataNode> DataNodeStore::load(unique_ref<Block> block) {
ASSERT(block->size() == _layout.blocksizeBytes(), "Loading block of wrong size");
DataNodeView node(std::move(block));
if (node.Depth() == 0) {
return make_unique_ref<DataLeafNode>(std::move(node));
} else if (node.Depth() <= MAX_DEPTH) {
return make_unique_ref<DataInnerNode>(std::move(node));
} else {
throw runtime_error("Tree is to deep. Data corruption?");
}
}
unique_ref<DataInnerNode> DataNodeStore::createNewInnerNode(const DataNode &first_child) {
ASSERT(first_child.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
//TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
return DataInnerNode::InitializeNewNode(std::move(block), first_child);
}
unique_ref<DataLeafNode> DataNodeStore::createNewLeafNode() {
//TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
return DataLeafNode::InitializeNewNode(std::move(block));
}
optional<unique_ref<DataNode>> DataNodeStore::load(const Key &key) {
auto block = _blockstore->load(key);
if (block == none) {
return none;
} else {
return load(std::move(*block));
}
}
unique_ref<DataNode> DataNodeStore::createNewNodeAsCopyFrom(const DataNode &source) {
ASSERT(source.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
auto newBlock = blockstore::utils::copyToNewBlock(_blockstore.get(), source.node().block());
return load(std::move(newBlock));
}
unique_ref<DataNode> DataNodeStore::overwriteNodeWith(unique_ref<DataNode> target, const DataNode &source) {
ASSERT(target->node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Target node has wrong layout. Is it from the same DataNodeStore?");
ASSERT(source.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
Key key = target->key();
{
auto targetBlock = target->node().releaseBlock();
cpputils::destruct(std::move(target)); // Call destructor
blockstore::utils::copyTo(targetBlock.get(), source.node().block());
}
auto loaded = load(key);
ASSERT(loaded != none, "Couldn't load the target node after overwriting it");
return std::move(*loaded);
}
void DataNodeStore::remove(unique_ref<DataNode> node) {
auto block = node->node().releaseBlock();
cpputils::destruct(std::move(node)); // Call destructor
_blockstore->remove(std::move(block));
}
uint64_t DataNodeStore::numNodes() const {
return _blockstore->numBlocks();
}
void DataNodeStore::removeSubtree(unique_ref<DataNode> node) {
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(node.get());
if (inner != nullptr) {
for (uint32_t i = 0; i < inner->numChildren(); ++i) {
auto child = load(inner->getChild(i)->key());
ASSERT(child != none, "Couldn't load child node");
removeSubtree(std::move(*child));
}
}
remove(std::move(node));
}
DataNodeLayout DataNodeStore::layout() const {
return _layout;
}
}
}
}

60
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h Normal file
View File

@ -0,0 +1,60 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODESTORE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODESTORE_H_
#include <memory>
#include <cpp-utils/macros.h>
#include "DataNodeView.h"
#include <blockstore/utils/Key.h>
namespace blockstore{
class Block;
class BlockStore;
}
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNode;
class DataLeafNode;
class DataInnerNode;
class DataNodeStore final {
public:
DataNodeStore(cpputils::unique_ref<blockstore::BlockStore> blockstore, uint32_t blocksizeBytes);
~DataNodeStore();
static constexpr uint8_t MAX_DEPTH = 10;
DataNodeLayout layout() const;
boost::optional<cpputils::unique_ref<DataNode>> load(const blockstore::Key &key);
cpputils::unique_ref<DataLeafNode> createNewLeafNode();
cpputils::unique_ref<DataInnerNode> createNewInnerNode(const DataNode &first_child);
cpputils::unique_ref<DataNode> createNewNodeAsCopyFrom(const DataNode &source);
cpputils::unique_ref<DataNode> overwriteNodeWith(cpputils::unique_ref<DataNode> target, const DataNode &source);
void remove(cpputils::unique_ref<DataNode> node);
void removeSubtree(cpputils::unique_ref<DataNode> node);
uint64_t numNodes() const;
//TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree()
private:
cpputils::unique_ref<DataNode> load(cpputils::unique_ref<blockstore::Block> block);
cpputils::unique_ref<blockstore::BlockStore> _blockstore;
const DataNodeLayout _layout;
DISALLOW_COPY_AND_ASSIGN(DataNodeStore);
};
}
}
}
#endif

140
src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h Normal file
View File

@ -0,0 +1,140 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODEVIEW_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODEVIEW_H_
#include <blockstore/interface/Block.h>
#include "../BlobStoreOnBlocks.h"
#include "DataInnerNode_ChildEntry.h"
#include <cpp-utils/pointer/unique_ref.h>
#include <memory>
#include <stdexcept>
#include <type_traits>
namespace blobstore {
namespace onblocks {
namespace datanodestore {
//TODO Move DataNodeLayout into own file
class DataNodeLayout final {
public:
constexpr DataNodeLayout(uint32_t blocksizeBytes)
:_blocksizeBytes(
(HEADERSIZE_BYTES + 2*sizeof(DataInnerNode_ChildEntry) <= blocksizeBytes)
? blocksizeBytes
: throw std::logic_error("Blocksize too small, not enough space to store two children in an inner node")) {
}
//Total size of the header
static constexpr uint32_t HEADERSIZE_BYTES = 8;
//Where in the header is the depth field
static constexpr uint32_t DEPTH_OFFSET_BYTES = 0;
//Where in the header is the size field (for inner nodes: number of children, for leafs: content data size)
static constexpr uint32_t SIZE_OFFSET_BYTES = 4;
//Size of a block (header + data region)
constexpr uint32_t blocksizeBytes() const {
return _blocksizeBytes;
}
//Number of bytes in the data region of a node
constexpr uint32_t datasizeBytes() const {
return _blocksizeBytes - HEADERSIZE_BYTES;
}
//Maximum number of children an inner node can store
constexpr uint32_t maxChildrenPerInnerNode() const {
return datasizeBytes() / sizeof(DataInnerNode_ChildEntry);
}
//Maximum number of bytes a leaf can store
//We are returning uint64_t here, because calculations involving maxBytesPerLeaf most probably should use 64bit integers to support blobs >4GB.
constexpr uint64_t maxBytesPerLeaf() const {
return datasizeBytes();
}
private:
uint32_t _blocksizeBytes;
};
class DataNodeView final {
public:
DataNodeView(cpputils::unique_ref<blockstore::Block> block): _block(std::move(block)) {
}
~DataNodeView() {}
DataNodeView(DataNodeView &&rhs) = default;
uint8_t Depth() const {
return *((uint8_t*)_block->data()+DataNodeLayout::DEPTH_OFFSET_BYTES);
}
void setDepth(uint8_t value) {
_block->write(&value, DataNodeLayout::DEPTH_OFFSET_BYTES, sizeof(value));
}
uint32_t Size() const {
return *(uint32_t*)((uint8_t*)_block->data()+DataNodeLayout::SIZE_OFFSET_BYTES);
}
void setSize(uint32_t value) {
_block->write(&value, DataNodeLayout::SIZE_OFFSET_BYTES, sizeof(value));
}
const void *data() const {
return (uint8_t*)_block->data() + DataNodeLayout::HEADERSIZE_BYTES;
}
void write(const void *source, uint64_t offset, uint64_t size) {
_block->write(source, offset + DataNodeLayout::HEADERSIZE_BYTES, size);
}
template<typename Entry>
const Entry *DataBegin() const {
return GetOffset<DataNodeLayout::HEADERSIZE_BYTES, Entry>();
}
template<typename Entry>
const Entry *DataEnd() const {
const unsigned int NUM_ENTRIES = layout().datasizeBytes() / sizeof(Entry);
return DataBegin<Entry>() + NUM_ENTRIES;
}
DataNodeLayout layout() const {
return DataNodeLayout(_block->size());
}
cpputils::unique_ref<blockstore::Block> releaseBlock() {
return std::move(_block);
}
const blockstore::Block &block() const {
return *_block;
}
const blockstore::Key &key() const {
return _block->key();
}
void flush() const {
_block->flush();
}
private:
template<int offset, class Type>
const Type *GetOffset() const {
return (Type*)(((const int8_t*)_block->data())+offset);
}
cpputils::unique_ref<blockstore::Block> _block;
DISALLOW_COPY_AND_ASSIGN(DataNodeView);
};
}
}
}
#endif

334
src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp Normal file
View File

@ -0,0 +1,334 @@
#include "DataTree.h"
#include "../datanodestore/DataNodeStore.h"
#include "../datanodestore/DataInnerNode.h"
#include "../datanodestore/DataLeafNode.h"
#include "../utils/Math.h"
#include "impl/algorithms.h"
#include <cpp-utils/pointer/cast.h>
#include <cpp-utils/pointer/optional_ownership_ptr.h>
#include <cmath>
#include <cpp-utils/assert/assert.h>
using blockstore::Key;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataNodeLayout;
using std::dynamic_pointer_cast;
using std::function;
using boost::shared_mutex;
using boost::shared_lock;
using boost::unique_lock;
using boost::none;
using std::vector;
using cpputils::dynamic_pointer_move;
using cpputils::optional_ownership_ptr;
using cpputils::WithOwnership;
using cpputils::WithoutOwnership;
using cpputils::unique_ref;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
DataTree::DataTree(DataNodeStore *nodeStore, unique_ref<DataNode> rootNode)
: _mutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)) {
}
DataTree::~DataTree() {
}
void DataTree::removeLastDataLeaf() {
auto deletePosOrNull = algorithms::GetLowestRightBorderNodeWithMoreThanOneChildOrNull(_nodeStore, _rootNode.get());
ASSERT(deletePosOrNull.get() != nullptr, "Tree has only one leaf, can't shrink it.");
deleteLastChildSubtree(deletePosOrNull.get());
ifRootHasOnlyOneChildReplaceRootWithItsChild();
}
void DataTree::ifRootHasOnlyOneChildReplaceRootWithItsChild() {
DataInnerNode *rootNode = dynamic_cast<DataInnerNode*>(_rootNode.get());
ASSERT(rootNode != nullptr, "RootNode is not an inner node");
if (rootNode->numChildren() == 1) {
auto child = _nodeStore->load(rootNode->getChild(0)->key());
ASSERT(child != none, "Couldn't load first child of root node");
_rootNode = _nodeStore->overwriteNodeWith(std::move(_rootNode), **child);
_nodeStore->remove(std::move(*child));
}
}
void DataTree::deleteLastChildSubtree(DataInnerNode *node) {
auto lastChild = _nodeStore->load(node->LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
_nodeStore->removeSubtree(std::move(*lastChild));
node->removeLastChild();
}
unique_ref<DataLeafNode> DataTree::addDataLeaf() {
auto insertPosOrNull = algorithms::GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(_nodeStore, _rootNode.get());
if (insertPosOrNull) {
return addDataLeafAt(insertPosOrNull.get());
} else {
return addDataLeafToFullTree();
}
}
unique_ref<DataLeafNode> DataTree::addDataLeafAt(DataInnerNode *insertPos) {
auto new_leaf = _nodeStore->createNewLeafNode();
auto chain = createChainOfInnerNodes(insertPos->depth()-1, new_leaf.get());
insertPos->addChild(*chain);
return new_leaf;
}
optional_ownership_ptr<DataNode> DataTree::createChainOfInnerNodes(unsigned int num, DataNode *child) {
//TODO This function is implemented twice, once with optional_ownership_ptr, once with unique_ref. Redundancy!
optional_ownership_ptr<DataNode> chain = cpputils::WithoutOwnership<DataNode>(child);
for(unsigned int i=0; i<num; ++i) {
auto newnode = _nodeStore->createNewInnerNode(*chain);
chain = cpputils::WithOwnership<DataNode>(std::move(newnode));
}
return chain;
}
unique_ref<DataNode> DataTree::createChainOfInnerNodes(unsigned int num, unique_ref<DataNode> child) {
unique_ref<DataNode> chain = std::move(child);
for(unsigned int i=0; i<num; ++i) {
chain = _nodeStore->createNewInnerNode(*chain);
}
return chain;
}
DataInnerNode* DataTree::increaseTreeDepth(unsigned int levels) {
ASSERT(levels >= 1, "Parameter out of bounds: tried to increase tree depth by zero.");
auto copyOfOldRoot = _nodeStore->createNewNodeAsCopyFrom(*_rootNode);
auto chain = createChainOfInnerNodes(levels-1, copyOfOldRoot.get());
auto newRootNode = DataNode::convertToNewInnerNode(std::move(_rootNode), *chain);
DataInnerNode *result = newRootNode.get();
_rootNode = std::move(newRootNode);
return result;
}
unique_ref<DataLeafNode> DataTree::addDataLeafToFullTree() {
DataInnerNode *rootNode = increaseTreeDepth(1);
auto newLeaf = addDataLeafAt(rootNode);
return newLeaf;
}
const Key &DataTree::key() const {
return _rootNode->key();
}
void DataTree::flush() const {
// By grabbing a lock, we ensure that all modifying functions don't run currently and are therefore flushed
unique_lock<shared_mutex> lock(_mutex);
// We also have to flush the root node
_rootNode->flush();
}
unique_ref<DataNode> DataTree::releaseRootNode() {
return std::move(_rootNode);
}
//TODO Test numLeaves(), for example also two configurations with same number of bytes but different number of leaves (last leaf has 0 bytes)
uint32_t DataTree::numLeaves() const {
shared_lock<shared_mutex> lock(_mutex);
return _numLeaves(*_rootNode);
}
uint32_t DataTree::_numLeaves(const DataNode &node) const {
const DataLeafNode *leaf = dynamic_cast<const DataLeafNode*>(&node);
if (leaf != nullptr) {
return 1;
}
const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(node);
uint64_t numLeavesInLeftChildren = (inner.numChildren()-1) * leavesPerFullChild(inner);
auto lastChild = _nodeStore->load(inner.LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
uint64_t numLeavesInRightChild = _numLeaves(**lastChild);
return numLeavesInLeftChildren + numLeavesInRightChild;
}
void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<void (DataLeafNode*, uint32_t)> func) {
//TODO Can we traverse in parallel?
unique_lock<shared_mutex> lock(_mutex); //TODO Only lock when resizing. Otherwise parallel read/write to a blob is not possible!
ASSERT(beginIndex <= endIndex, "Invalid parameters");
if (0 == endIndex) {
// In this case the utils::ceilLog(_, endIndex) below would fail
return;
}
uint8_t neededTreeDepth = utils::ceilLog(_nodeStore->layout().maxChildrenPerInnerNode(), endIndex);
uint32_t numLeaves = this->_numLeaves(*_rootNode); // TODO Querying the size causes a tree traversal down to the leaves. Possible without querying the size?
if (_rootNode->depth() < neededTreeDepth) {
//TODO Test cases that actually increase it here by 0 level / 1 level / more than 1 level
increaseTreeDepth(neededTreeDepth - _rootNode->depth());
}
if (numLeaves <= beginIndex) {
//TODO Test cases with numLeaves < / >= beginIndex
// There is a gap between the current size and the begin of the traversal
return _traverseLeaves(_rootNode.get(), 0, numLeaves-1, endIndex, [beginIndex, numLeaves, &func, this](DataLeafNode* node, uint32_t index) {
if (index >= beginIndex) {
func(node, index);
} else if (index == numLeaves - 1) {
// It is the old last leaf - resize it to maximum
node->resize(_nodeStore->layout().maxBytesPerLeaf());
}
});
} else if (numLeaves < endIndex) {
// We are starting traversal in the valid region, but traverse until after it (we grow new leaves)
return _traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, [numLeaves, &func, this] (DataLeafNode *node, uint32_t index) {
if (index == numLeaves - 1) {
// It is the old last leaf - resize it to maximum
node->resize(_nodeStore->layout().maxBytesPerLeaf());
}
func(node, index);
});
} else {
//We are traversing entirely inside the valid region
_traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, func);
}
}
void DataTree::_traverseLeaves(DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, function<void (DataLeafNode*, uint32_t)> func) {
DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root);
if (leaf != nullptr) {
ASSERT(beginIndex <= 1 && endIndex <= 1, "If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1.");
if (beginIndex == 0 && endIndex == 1) {
func(leaf, leafOffset);
}
return;
}
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root);
uint32_t leavesPerChild = leavesPerFullChild(*inner);
uint32_t beginChild = beginIndex/leavesPerChild;
uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild);
vector<unique_ref<DataNode>> children = getOrCreateChildren(inner, beginChild, endChild);
for (uint32_t childIndex = beginChild; childIndex < endChild; ++childIndex) {
uint32_t childOffset = childIndex * leavesPerChild;
uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset);
uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
auto child = std::move(children[childIndex-beginChild]);
_traverseLeaves(child.get(), leafOffset + childOffset, localBeginIndex, localEndIndex, func);
}
}
vector<unique_ref<DataNode>> DataTree::getOrCreateChildren(DataInnerNode *node, uint32_t begin, uint32_t end) {
vector<unique_ref<DataNode>> children;
children.reserve(end-begin);
for (uint32_t childIndex = begin; childIndex < std::min(node->numChildren(), end); ++childIndex) {
auto child = _nodeStore->load(node->getChild(childIndex)->key());
ASSERT(child != none, "Couldn't load child node");
children.emplace_back(std::move(*child));
}
for (uint32_t childIndex = node->numChildren(); childIndex < end; ++childIndex) {
//TODO This creates each child with one chain to one leaf only, and then on the next lower level it
// has to create the children for the child. Would be faster to directly create full trees if necessary.
children.emplace_back(addChildTo(node));
}
ASSERT(children.size() == end-begin, "Number of children in the result is wrong");
return children;
}
unique_ref<DataNode> DataTree::addChildTo(DataInnerNode *node) {
auto new_leaf = _nodeStore->createNewLeafNode();
new_leaf->resize(_nodeStore->layout().maxBytesPerLeaf());
auto chain = createChainOfInnerNodes(node->depth()-1, std::move(new_leaf));
node->addChild(*chain);
return std::move(chain);
}
uint32_t DataTree::leavesPerFullChild(const DataInnerNode &root) const {
return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), (uint32_t)root.depth()-1);
}
uint64_t DataTree::numStoredBytes() const {
shared_lock<shared_mutex> lock(_mutex);
return _numStoredBytes();
}
uint64_t DataTree::_numStoredBytes() const {
return _numStoredBytes(*_rootNode);
}
uint64_t DataTree::_numStoredBytes(const DataNode &root) const {
const DataLeafNode *leaf = dynamic_cast<const DataLeafNode*>(&root);
if (leaf != nullptr) {
return leaf->numBytes();
}
const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(root);
uint64_t numBytesInLeftChildren = (inner.numChildren()-1) * leavesPerFullChild(inner) * _nodeStore->layout().maxBytesPerLeaf();
auto lastChild = _nodeStore->load(inner.LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
uint64_t numBytesInRightChild = _numStoredBytes(**lastChild);
return numBytesInLeftChildren + numBytesInRightChild;
}
void DataTree::resizeNumBytes(uint64_t newNumBytes) {
//TODO Can we resize in parallel? Especially creating new blocks (i.e. encrypting them) is expensive and should be done in parallel.
boost::upgrade_lock<shared_mutex> lock(_mutex);
{
boost::upgrade_to_unique_lock<shared_mutex> exclusiveLock(lock);
//TODO Faster implementation possible (no addDataLeaf()/removeLastDataLeaf() in a loop, but directly resizing)
LastLeaf(_rootNode.get())->resize(_nodeStore->layout().maxBytesPerLeaf());
uint64_t currentNumBytes = _numStoredBytes();
ASSERT(currentNumBytes % _nodeStore->layout().maxBytesPerLeaf() == 0, "The last leaf is not a max data leaf, although we just resized it to be one.");
uint32_t currentNumLeaves = currentNumBytes / _nodeStore->layout().maxBytesPerLeaf();
uint32_t newNumLeaves = std::max(UINT64_C(1), utils::ceilDivision(newNumBytes, _nodeStore->layout().maxBytesPerLeaf()));
for(uint32_t i = currentNumLeaves; i < newNumLeaves; ++i) {
addDataLeaf()->resize(_nodeStore->layout().maxBytesPerLeaf());
}
for(uint32_t i = currentNumLeaves; i > newNumLeaves; --i) {
removeLastDataLeaf();
}
uint32_t newLastLeafSize = newNumBytes - (newNumLeaves-1)*_nodeStore->layout().maxBytesPerLeaf();
LastLeaf(_rootNode.get())->resize(newLastLeafSize);
}
ASSERT(newNumBytes == _numStoredBytes(), "We resized to the wrong number of bytes ("+std::to_string(numStoredBytes())+" instead of "+std::to_string(newNumBytes)+")");
}
optional_ownership_ptr<DataLeafNode> DataTree::LastLeaf(DataNode *root) {
DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root);
if (leaf != nullptr) {
return WithoutOwnership(leaf);
}
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root);
auto lastChild = _nodeStore->load(inner->LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
return WithOwnership(LastLeaf(std::move(*lastChild)));
}
unique_ref<DataLeafNode> DataTree::LastLeaf(unique_ref<DataNode> root) {
auto leaf = dynamic_pointer_move<DataLeafNode>(root);
if (leaf != none) {
return std::move(*leaf);
}
auto inner = dynamic_pointer_move<DataInnerNode>(root);
ASSERT(inner != none, "Root node is neither a leaf nor an inner node");
auto child = _nodeStore->load((*inner)->LastChild()->key());
ASSERT(child != none, "Couldn't load last child");
return LastLeaf(std::move(*child));
}
uint64_t DataTree::maxBytesPerLeaf() const {
return _nodeStore->layout().maxBytesPerLeaf();
}
}
}
}

79
src/blobstore/implementations/onblocks/datatreestore/DataTree.h Normal file
View File

@ -0,0 +1,79 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREE_H_
#include <memory>
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/optional_ownership_ptr.h>
#include "../datanodestore/DataNodeView.h"
//TODO Replace with C++14 once std::shared_mutex is supported
#include <boost/thread/shared_mutex.hpp>
#include <blockstore/utils/Key.h>
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNodeStore;
class DataInnerNode;
class DataLeafNode;
class DataNode;
}
namespace datatreestore {
//TODO It is strange that DataLeafNode is still part in the public interface of DataTree. This should be separated somehow.
class DataTree final {
public:
DataTree(datanodestore::DataNodeStore *nodeStore, cpputils::unique_ref<datanodestore::DataNode> rootNode);
~DataTree();
const blockstore::Key &key() const;
//Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
uint64_t maxBytesPerLeaf() const;
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func);
void resizeNumBytes(uint64_t newNumBytes);
uint32_t numLeaves() const;
uint64_t numStoredBytes() const;
void flush() const;
private:
mutable boost::shared_mutex _mutex;
datanodestore::DataNodeStore *_nodeStore;
cpputils::unique_ref<datanodestore::DataNode> _rootNode;
cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeaf();
void removeLastDataLeaf();
cpputils::unique_ref<datanodestore::DataNode> releaseRootNode();
friend class DataTreeStore;
cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeafAt(datanodestore::DataInnerNode *insertPos);
cpputils::optional_ownership_ptr<datanodestore::DataNode> createChainOfInnerNodes(unsigned int num, datanodestore::DataNode *child);
cpputils::unique_ref<datanodestore::DataNode> createChainOfInnerNodes(unsigned int num, cpputils::unique_ref<datanodestore::DataNode> child);
cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeafToFullTree();
void deleteLastChildSubtree(datanodestore::DataInnerNode *node);
void ifRootHasOnlyOneChildReplaceRootWithItsChild();
//TODO Use underscore for private methods
void _traverseLeaves(datanodestore::DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func);
uint32_t leavesPerFullChild(const datanodestore::DataInnerNode &root) const;
uint64_t _numStoredBytes() const;
uint64_t _numStoredBytes(const datanodestore::DataNode &root) const;
uint32_t _numLeaves(const datanodestore::DataNode &node) const;
cpputils::optional_ownership_ptr<datanodestore::DataLeafNode> LastLeaf(datanodestore::DataNode *root);
cpputils::unique_ref<datanodestore::DataLeafNode> LastLeaf(cpputils::unique_ref<datanodestore::DataNode> root);
datanodestore::DataInnerNode* increaseTreeDepth(unsigned int levels);
std::vector<cpputils::unique_ref<datanodestore::DataNode>> getOrCreateChildren(datanodestore::DataInnerNode *node, uint32_t begin, uint32_t end);
cpputils::unique_ref<datanodestore::DataNode> addChildTo(datanodestore::DataInnerNode *node);
DISALLOW_COPY_AND_ASSIGN(DataTree);
};
}
}
}
#endif

46
src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.cpp Normal file
View File

@ -0,0 +1,46 @@
#include "DataTreeStore.h"
#include "../datanodestore/DataNodeStore.h"
#include "../datanodestore/DataLeafNode.h"
#include "DataTree.h"
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blobstore::onblocks::datanodestore::DataNode;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
DataTreeStore::DataTreeStore(unique_ref<DataNodeStore> nodeStore)
: _nodeStore(std::move(nodeStore)) {
}
DataTreeStore::~DataTreeStore() {
}
optional<unique_ref<DataTree>> DataTreeStore::load(const blockstore::Key &key) {
auto node = _nodeStore->load(key);
if (node == none) {
return none;
}
return make_unique_ref<DataTree>(_nodeStore.get(), std::move(*node));
}
unique_ref<DataTree> DataTreeStore::createNewTree() {
auto newleaf = _nodeStore->createNewLeafNode();
return make_unique_ref<DataTree>(_nodeStore.get(), std::move(newleaf));
}
void DataTreeStore::remove(unique_ref<DataTree> tree) {
auto root = tree->releaseRootNode();
cpputils::destruct(std::move(tree)); // Destruct tree
_nodeStore->removeSubtree(std::move(root));
}
}
}
}

40
src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h Normal file
View File

@ -0,0 +1,40 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_DATATREESTORE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_DATATREESTORE_H_
#include <memory>
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/unique_ref.h>
#include <blockstore/utils/Key.h>
#include <boost/optional.hpp>
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNodeStore;
}
namespace datatreestore {
class DataTree;
class DataTreeStore final {
public:
DataTreeStore(cpputils::unique_ref<datanodestore::DataNodeStore> nodeStore);
~DataTreeStore();
boost::optional<cpputils::unique_ref<DataTree>> load(const blockstore::Key &key);
cpputils::unique_ref<DataTree> createNewTree();
void remove(cpputils::unique_ref<DataTree> tree);
private:
cpputils::unique_ref<datanodestore::DataNodeStore> _nodeStore;
DISALLOW_COPY_AND_ASSIGN(DataTreeStore);
};
}
}
}
#endif

67
src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.cpp Normal file
View File

@ -0,0 +1,67 @@
#include "algorithms.h"
#include <cpp-utils/pointer/cast.h>
#include <blockstore/utils/Key.h>
#include "../../datanodestore/DataInnerNode.h"
#include "../../datanodestore/DataNodeStore.h"
#include <cpp-utils/assert/assert.h>
using std::function;
using cpputils::optional_ownership_ptr;
using cpputils::dynamic_pointer_move;
using cpputils::unique_ref;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blockstore::Key;
using boost::optional;
using boost::none;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
namespace algorithms {
optional<unique_ref<DataInnerNode>> getLastChildAsInnerNode(DataNodeStore *nodeStore, const DataInnerNode &node) {
Key key = node.LastChild()->key();
auto lastChild = nodeStore->load(key);
ASSERT(lastChild != none, "Couldn't load last child");
return dynamic_pointer_move<DataInnerNode>(*lastChild);
}
//Returns the lowest right border node meeting the condition specified (exclusive the leaf).
//Returns nullptr, if no inner right border node meets the condition.
optional_ownership_ptr<DataInnerNode> GetLowestInnerRightBorderNodeWithConditionOrNull(DataNodeStore *nodeStore, datanodestore::DataNode *rootNode, function<bool(const DataInnerNode &)> condition) {
optional_ownership_ptr<DataInnerNode> currentNode = cpputils::WithoutOwnership(dynamic_cast<DataInnerNode*>(rootNode));
optional_ownership_ptr<DataInnerNode> result = cpputils::null<DataInnerNode>();
for (unsigned int i=0; i < rootNode->depth(); ++i) {
//TODO This unnecessarily loads the leaf node in the last loop run
auto lastChild = getLastChildAsInnerNode(nodeStore, *currentNode);
if (condition(*currentNode)) {
result = std::move(currentNode);
}
ASSERT(lastChild != none || static_cast<int>(i) == rootNode->depth()-1, "Couldn't get last child as inner node but we're not deep enough yet for the last child to be a leaf");
if (lastChild != none) {
currentNode = cpputils::WithOwnership(std::move(*lastChild));
}
}
return result;
}
optional_ownership_ptr<DataInnerNode> GetLowestRightBorderNodeWithMoreThanOneChildOrNull(DataNodeStore *nodeStore, DataNode *rootNode) {
return GetLowestInnerRightBorderNodeWithConditionOrNull(nodeStore, rootNode, [] (const datanodestore::DataInnerNode &node) {
return node.numChildren() > 1;
});
}
optional_ownership_ptr<datanodestore::DataInnerNode> GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode) {
return GetLowestInnerRightBorderNodeWithConditionOrNull(nodeStore, rootNode, [] (const datanodestore::DataInnerNode &node) {
return node.numChildren() < node.maxStoreableChildren();
});
}
}
}
}
}

30
src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.h Normal file
View File

@ -0,0 +1,30 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_IMPL_ALGORITHMS_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_IMPL_ALGORITHMS_H_
#include <cpp-utils/pointer/optional_ownership_ptr.h>
namespace blobstore {
namespace onblocks {
namespace datanodestore{
class DataNode;
class DataInnerNode;
class DataNodeStore;
}
namespace datatreestore {
namespace algorithms {
//Returns the lowest right border node with at least two children.
//Returns nullptr, if all right border nodes have only one child (since the root is a right border node, this means that the whole tree has exactly one leaf)
cpputils::optional_ownership_ptr<datanodestore::DataInnerNode> GetLowestRightBorderNodeWithMoreThanOneChildOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode);
//Returns the lowest right border node with less than k children (not considering leaves).
//Returns nullptr, if all right border nodes have k children (the tree is full)
cpputils::optional_ownership_ptr<datanodestore::DataInnerNode> GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode);
}
}
}
}
#endif

1
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.cpp Normal file
View File

@ -0,0 +1 @@
#include "DataTreeRef.h"

55
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h Normal file
View File

@ -0,0 +1,55 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_DATATREEREF_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_DATATREEREF_H_
#include <parallelaccessstore/ParallelAccessStore.h>
#include "../datatreestore/DataTree.h"
namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class DataTreeRef final: public parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::Key>::ResourceRefBase {
public:
DataTreeRef(datatreestore::DataTree *baseTree): _baseTree(baseTree) {}
const blockstore::Key &key() const {
return _baseTree->key();
}
uint64_t maxBytesPerLeaf() const {
return _baseTree->maxBytesPerLeaf();
}
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func) {
return _baseTree->traverseLeaves(beginIndex, endIndex, func);
}
uint32_t numLeaves() const {
return _baseTree->numLeaves();
}
void resizeNumBytes(uint64_t newNumBytes) {
return _baseTree->resizeNumBytes(newNumBytes);
}
uint64_t numStoredBytes() const {
return _baseTree->numStoredBytes();
}
void flush() {
return _baseTree->flush();
}
private:
datatreestore::DataTree *_baseTree;
DISALLOW_COPY_AND_ASSIGN(DataTreeRef);
};
}
}
}
#endif

44
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp Normal file
View File

@ -0,0 +1,44 @@
#include "DataTreeRef.h"
#include "ParallelAccessDataTreeStore.h"
#include "ParallelAccessDataTreeStoreAdapter.h"
#include "../datanodestore/DataNodeStore.h"
#include "../datanodestore/DataLeafNode.h"
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using blobstore::onblocks::datatreestore::DataTreeStore;
using blockstore::Key;
namespace blobstore {
namespace onblocks {
using datatreestore::DataTreeStore;
using datatreestore::DataTree;
namespace parallelaccessdatatreestore {
ParallelAccessDataTreeStore::ParallelAccessDataTreeStore(unique_ref<DataTreeStore> dataTreeStore)
: _dataTreeStore(std::move(dataTreeStore)), _parallelAccessStore(make_unique_ref<ParallelAccessDataTreeStoreAdapter>(_dataTreeStore.get())) {
}
ParallelAccessDataTreeStore::~ParallelAccessDataTreeStore() {
}
optional<unique_ref<DataTreeRef>> ParallelAccessDataTreeStore::load(const blockstore::Key &key) {
return _parallelAccessStore.load(key);
}
unique_ref<DataTreeRef> ParallelAccessDataTreeStore::createNewTree() {
auto dataTree = _dataTreeStore->createNewTree();
Key key = dataTree->key();
return _parallelAccessStore.add(key, std::move(dataTree));
}
void ParallelAccessDataTreeStore::remove(unique_ref<DataTreeRef> tree) {
Key key = tree->key();
return _parallelAccessStore.remove(key, std::move(tree));
}
}
}
}

43
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h Normal file
View File

@ -0,0 +1,43 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTORE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTORE_H_
#include <memory>
#include <cpp-utils/macros.h>
#include <blockstore/utils/Key.h>
#include <parallelaccessstore/ParallelAccessStore.h>
namespace blobstore {
namespace onblocks {
namespace datatreestore {
class DataTreeStore;
class DataTree;
}
namespace parallelaccessdatatreestore {
class DataTreeRef;
//TODO Test CachingDataTreeStore
class ParallelAccessDataTreeStore final {
public:
ParallelAccessDataTreeStore(cpputils::unique_ref<datatreestore::DataTreeStore> dataTreeStore);
~ParallelAccessDataTreeStore();
boost::optional<cpputils::unique_ref<DataTreeRef>> load(const blockstore::Key &key);
cpputils::unique_ref<DataTreeRef> createNewTree();
void remove(cpputils::unique_ref<DataTreeRef> tree);
private:
cpputils::unique_ref<datatreestore::DataTreeStore> _dataTreeStore;
parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::Key> _parallelAccessStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessDataTreeStore);
};
}
}
}
#endif

1
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.cpp Normal file
View File

@ -0,0 +1 @@
#include "ParallelAccessDataTreeStoreAdapter.h"

38
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.h Normal file
View File

@ -0,0 +1,38 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTOREADAPTER_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTOREADAPTER_H_
#include <cpp-utils/macros.h>
#include <parallelaccessstore/ParallelAccessStore.h>
#include "../datatreestore/DataTreeStore.h"
#include "../datatreestore/DataTree.h"
namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class ParallelAccessDataTreeStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore<datatreestore::DataTree, blockstore::Key> {
public:
ParallelAccessDataTreeStoreAdapter(datatreestore::DataTreeStore *baseDataTreeStore)
:_baseDataTreeStore(std::move(baseDataTreeStore)) {
}
boost::optional<cpputils::unique_ref<datatreestore::DataTree>> loadFromBaseStore(const blockstore::Key &key) override {
return _baseDataTreeStore->load(key);
}
void removeFromBaseStore(cpputils::unique_ref<datatreestore::DataTree> dataTree) override {
return _baseDataTreeStore->remove(std::move(dataTree));
}
private:
datatreestore::DataTreeStore *_baseDataTreeStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessDataTreeStoreAdapter);
};
}
}
}
#endif

1
src/blobstore/implementations/onblocks/utils/Math.cpp Normal file
View File

@ -0,0 +1 @@
#include "Math.h"

43
src/blobstore/implementations/onblocks/utils/Math.h Normal file
View File

@ -0,0 +1,43 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_UTILS_MATH_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_UTILS_MATH_H_
#include <cstdint>
#include <cmath>
namespace blobstore {
namespace onblocks {
namespace utils {
template<typename INT_TYPE>
inline INT_TYPE intPow(INT_TYPE base, INT_TYPE exponent) {
INT_TYPE result = 1;
for(INT_TYPE i = 0; i < exponent; ++i) {
result *= base;
}
return result;
}
template<typename INT_TYPE>
inline INT_TYPE ceilDivision(INT_TYPE dividend, INT_TYPE divisor) {
return (dividend + divisor - 1)/divisor;
}
template<typename INT_TYPE>
inline INT_TYPE maxZeroSubtraction(INT_TYPE minuend, INT_TYPE subtrahend) {
if (minuend < subtrahend) {
return 0u;
}
return minuend-subtrahend;
}
template<typename INT_TYPE>
inline INT_TYPE ceilLog(INT_TYPE base, INT_TYPE value) {
return std::ceil((long double)std::log(value)/(long double)std::log(base));
}
}
}
}
#endif

35
src/blobstore/interface/Blob.h Normal file
View File

@ -0,0 +1,35 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_INTERFACE_BLOB_H_
#define MESSMER_BLOBSTORE_INTERFACE_BLOB_H_
#include <cstring>
#include <cstdint>
#include <blockstore/utils/Key.h>
#include <cpp-utils/data/Data.h>
namespace blobstore {
class Blob {
public:
virtual ~Blob() {}
//TODO Use own Key class for blobstore
virtual const blockstore::Key &key() const = 0;
virtual uint64_t size() const = 0;
virtual void resize(uint64_t numBytes) = 0;
virtual cpputils::Data readAll() const = 0;
virtual void read(void *target, uint64_t offset, uint64_t size) const = 0;
virtual uint64_t tryRead(void *target, uint64_t offset, uint64_t size) const = 0;
virtual void write(const void *source, uint64_t offset, uint64_t size) = 0;
virtual void flush() = 0;
//TODO Test tryRead
};
}
#endif

25
src/blobstore/interface/BlobStore.h Normal file
View File

@ -0,0 +1,25 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_INTERFACE_BLOBSTORE_H_
#define MESSMER_BLOBSTORE_INTERFACE_BLOBSTORE_H_
#include "Blob.h"
#include <string>
#include <memory>
#include <blockstore/utils/Key.h>
#include <cpp-utils/pointer/unique_ref.h>
namespace blobstore {
class BlobStore {
public:
virtual ~BlobStore() {}
virtual cpputils::unique_ref<Blob> create() = 0;
virtual boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::Key &key) = 0;
virtual void remove(cpputils::unique_ref<Blob> blob) = 0;
};
}
#endif

43
src/blockstore/CMakeLists.txt Normal file
View File

@ -0,0 +1,43 @@
project (blockstore)
set(SOURCES
utils/Key.cpp
utils/BlockStoreUtils.cpp
utils/FileDoesntExistException.cpp
interface/helpers/BlockStoreWithRandomKeys.cpp
implementations/testfake/FakeBlockStore.cpp
implementations/testfake/FakeBlock.cpp
implementations/inmemory/InMemoryBlock.cpp
implementations/inmemory/InMemoryBlockStore.cpp
implementations/parallelaccess/ParallelAccessBlockStore.cpp
implementations/parallelaccess/BlockRef.cpp
implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp
implementations/compressing/CompressingBlockStore.cpp
implementations/compressing/CompressedBlock.cpp
implementations/compressing/compressors/RunLengthEncoding.cpp
implementations/compressing/compressors/Gzip.cpp
implementations/encrypted/EncryptedBlockStore.cpp
implementations/encrypted/EncryptedBlock.cpp
implementations/ondisk/OnDiskBlockStore.cpp
implementations/ondisk/OnDiskBlock.cpp
implementations/caching/CachingBlockStore.cpp
implementations/caching/cache/PeriodicTask.cpp
implementations/caching/cache/CacheEntry.cpp
implementations/caching/cache/Cache.cpp
implementations/caching/cache/QueueMap.cpp
implementations/caching/CachedBlock.cpp
implementations/caching/NewBlock.cpp
)
add_library(${PROJECT_NAME} STATIC ${SOURCES})
# This is needed by boost thread
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PRIVATE rt)
endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PUBLIC cpp-utils)
target_add_boost(${PROJECT_NAME} filesystem system thread)
target_enable_style_warnings(${PROJECT_NAME})
target_activate_cpp14(${PROJECT_NAME})

46
src/blockstore/implementations/caching/CachedBlock.cpp Normal file
View File

@ -0,0 +1,46 @@
#include "CachedBlock.h"
#include "CachingBlockStore.h"
using cpputils::unique_ref;
namespace blockstore {
namespace caching {
CachedBlock::CachedBlock(unique_ref<Block> baseBlock, CachingBlockStore *blockStore)
:Block(baseBlock->key()),
_blockStore(blockStore),
_baseBlock(std::move(baseBlock)) {
}
CachedBlock::~CachedBlock() {
if (_baseBlock.get() != nullptr) {
_blockStore->release(std::move(_baseBlock));
}
}
const void *CachedBlock::data() const {
return _baseBlock->data();
}
void CachedBlock::write(const void *source, uint64_t offset, uint64_t size) {
return _baseBlock->write(source, offset, size);
}
void CachedBlock::flush() {
return _baseBlock->flush();
}
size_t CachedBlock::size() const {
return _baseBlock->size();
}
void CachedBlock::resize(size_t newSize) {
return _baseBlock->resize(newSize);
}
unique_ref<Block> CachedBlock::releaseBlock() {
return std::move(_baseBlock);
}
}
}

39
src/blockstore/implementations/caching/CachedBlock.h Normal file
View File

@ -0,0 +1,39 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/pointer/unique_ref.h>
namespace blockstore {
namespace caching {
class CachingBlockStore;
class CachedBlock final: public Block {
public:
//TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
CachedBlock(cpputils::unique_ref<Block> baseBlock, CachingBlockStore *blockStore);
~CachedBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref<Block> releaseBlock();
private:
CachingBlockStore *_blockStore;
cpputils::unique_ref<Block> _baseBlock;
DISALLOW_COPY_AND_ASSIGN(CachedBlock);
};
}
}
#endif

92
src/blockstore/implementations/caching/CachingBlockStore.cpp Normal file
View File

@ -0,0 +1,92 @@
#include "CachedBlock.h"
#include "NewBlock.h"
#include "CachingBlockStore.h"
#include "../../interface/Block.h"
#include <algorithm>
#include <cpp-utils/pointer/cast.h>
#include <cpp-utils/assert/assert.h>
using cpputils::dynamic_pointer_move;
using cpputils::Data;
using boost::optional;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::none;
namespace blockstore {
namespace caching {
CachingBlockStore::CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore)
:_baseBlockStore(std::move(baseBlockStore)), _cache(), _numNewBlocks(0) {
}
Key CachingBlockStore::createKey() {
return _baseBlockStore->createKey();
}
optional<unique_ref<Block>> CachingBlockStore::tryCreate(const Key &key, Data data) {
ASSERT(_cache.pop(key) == none, "Key already exists in cache");
//TODO Shouldn't we return boost::none if the key already exists?
//TODO Key can also already exist but not be in the cache right now.
++_numNewBlocks;
return unique_ref<Block>(make_unique_ref<CachedBlock>(make_unique_ref<NewBlock>(key, std::move(data), this), this));
}
optional<unique_ref<Block>> CachingBlockStore::load(const Key &key) {
optional<unique_ref<Block>> optBlock = _cache.pop(key);
//TODO an optional<> class with .getOrElse() would make this code simpler. boost::optional<>::value_or_eval didn't seem to work with unique_ptr members.
if (optBlock != none) {
return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*optBlock), this));
} else {
auto block = _baseBlockStore->load(key);
if (block == none) {
return none;
} else {
return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*block), this));
}
}
}
void CachingBlockStore::remove(cpputils::unique_ref<Block> block) {
auto cached_block = dynamic_pointer_move<CachedBlock>(block);
ASSERT(cached_block != none, "Passed block is not a CachedBlock");
auto baseBlock = (*cached_block)->releaseBlock();
auto baseNewBlock = dynamic_pointer_move<NewBlock>(baseBlock);
if (baseNewBlock != none) {
if(!(*baseNewBlock)->alreadyExistsInBaseStore()) {
--_numNewBlocks;
}
(*baseNewBlock)->remove();
} else {
_baseBlockStore->remove(std::move(baseBlock));
}
}
uint64_t CachingBlockStore::numBlocks() const {
return _baseBlockStore->numBlocks() + _numNewBlocks;
}
void CachingBlockStore::release(unique_ref<Block> block) {
Key key = block->key();
_cache.push(key, std::move(block));
}
optional<unique_ref<Block>> CachingBlockStore::tryCreateInBaseStore(const Key &key, Data data) {
auto block = _baseBlockStore->tryCreate(key, std::move(data));
if (block != none) {
--_numNewBlocks;
}
return block;
}
void CachingBlockStore::removeFromBaseStore(cpputils::unique_ref<Block> block) {
_baseBlockStore->remove(std::move(block));
}
void CachingBlockStore::flush() {
_cache.flush();
}
}
}

40
src/blockstore/implementations/caching/CachingBlockStore.h Normal file
View File

@ -0,0 +1,40 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
#include "cache/Cache.h"
#include "../../interface/BlockStore.h"
namespace blockstore {
namespace caching {
//TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel)
class CachingBlockStore final: public BlockStore {
public:
explicit CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
void release(cpputils::unique_ref<Block> block);
boost::optional<cpputils::unique_ref<Block>> tryCreateInBaseStore(const Key &key, cpputils::Data data);
void removeFromBaseStore(cpputils::unique_ref<Block> block);
void flush();
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
Cache<Key, cpputils::unique_ref<Block>, 1000> _cache;
uint32_t _numNewBlocks;
DISALLOW_COPY_AND_ASSIGN(CachingBlockStore);
};
}
}
#endif

75
src/blockstore/implementations/caching/NewBlock.cpp Normal file
View File

@ -0,0 +1,75 @@
#include "NewBlock.h"
#include "CachingBlockStore.h"
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
using cpputils::Data;
using boost::none;
namespace blockstore {
namespace caching {
NewBlock::NewBlock(const Key &key, Data data, CachingBlockStore *blockStore)
:Block(key),
_blockStore(blockStore),
_data(std::move(data)),
_baseBlock(none),
_dataChanged(true) {
}
NewBlock::~NewBlock() {
writeToBaseBlockIfChanged();
}
const void *NewBlock::data() const {
return _data.data();
}
void NewBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area");
std::memcpy((uint8_t*)_data.data()+offset, source, size);
_dataChanged = true;
}
void NewBlock::writeToBaseBlockIfChanged() {
if (_dataChanged) {
if (_baseBlock == none) {
//TODO _data.copy() necessary?
auto newBase = _blockStore->tryCreateInBaseStore(key(), _data.copy());
ASSERT(newBase != boost::none, "Couldn't create base block"); //TODO What if tryCreate fails due to a duplicate key? We should ensure we don't use duplicate keys.
_baseBlock = std::move(*newBase);
} else {
(*_baseBlock)->write(_data.data(), 0, _data.size());
}
_dataChanged = false;
}
}
void NewBlock::remove() {
if (_baseBlock != none) {
_blockStore->removeFromBaseStore(std::move(*_baseBlock));
}
_dataChanged = false;
}
void NewBlock::flush() {
writeToBaseBlockIfChanged();
ASSERT(_baseBlock != none, "At this point, the base block should already have been created but wasn't");
(*_baseBlock)->flush();
}
size_t NewBlock::size() const {
return _data.size();
}
void NewBlock::resize(size_t newSize) {
_data = cpputils::DataUtils::resize(std::move(_data), newSize);
_dataChanged = true;
}
bool NewBlock::alreadyExistsInBaseStore() const {
return _baseBlock != none;
}
}
}

52
src/blockstore/implementations/caching/NewBlock.h Normal file
View File

@ -0,0 +1,52 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
#include "../../interface/BlockStore.h"
#include <cpp-utils/data/Data.h>
#include <cpp-utils/macros.h>
#include <memory>
namespace blockstore {
namespace caching {
class CachingBlockStore;
//TODO Does it make sense to write a general DataBackedBlock that just stores a Data object and maps the block operations to it?
// Can we reuse that object somewhere else?
// Maybe a second abstract class for BlockRefBackedBlock?
// This is a block that was created in CachingBlockStore, but doesn't exist in the base block store yet.
// It only exists in the cache and it is created in the base block store when destructed.
class NewBlock final: public Block {
public:
NewBlock(const Key &key, cpputils::Data data, CachingBlockStore *blockStore);
~NewBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
void remove();
bool alreadyExistsInBaseStore() const;
private:
CachingBlockStore *_blockStore;
cpputils::Data _data;
boost::optional<cpputils::unique_ref<Block>> _baseBlock;
bool _dataChanged;
void writeToBaseBlockIfChanged();
DISALLOW_COPY_AND_ASSIGN(NewBlock);
};
}
}
#endif

1
src/blockstore/implementations/caching/cache/Cache.cpp vendored Normal file
View File

@ -0,0 +1 @@
#include "Cache.h"

178
src/blockstore/implementations/caching/cache/Cache.h vendored Normal file
View File

@ -0,0 +1,178 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHE_H_
#include "CacheEntry.h"
#include "QueueMap.h"
#include "PeriodicTask.h"
#include <memory>
#include <boost/optional.hpp>
#include <future>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/lock/MutexPoolLock.h>
#include <cpp-utils/pointer/gcc_4_8_compatibility.h>
namespace blockstore {
namespace caching {
template<class Key, class Value, uint32_t MAX_ENTRIES>
class Cache final {
public:
//TODO Current MAX_LIFETIME_SEC only considers time since the element was last pushed to the Cache. Also insert a real MAX_LIFETIME_SEC that forces resync of entries that have been pushed/popped often (e.g. the root blob)
//TODO Experiment with good values
static constexpr double PURGE_LIFETIME_SEC = 0.5; //When an entry has this age, it will be purged from the cache
static constexpr double PURGE_INTERVAL = 0.5; // With this interval, we check for entries to purge
static constexpr double MAX_LIFETIME_SEC = PURGE_LIFETIME_SEC + PURGE_INTERVAL; // This is the oldest age an entry can reach (given purging works in an ideal world, i.e. with the ideal interval and in zero time)
Cache();
~Cache();
uint32_t size() const;
void push(const Key &key, Value value);
boost::optional<Value> pop(const Key &key);
void flush();
private:
void _makeSpaceForEntry(std::unique_lock<std::mutex> *lock);
void _deleteEntry(std::unique_lock<std::mutex> *lock);
void _deleteOldEntriesParallel();
void _deleteAllEntriesParallel();
void _deleteMatchingEntriesAtBeginningParallel(std::function<bool (const CacheEntry<Key, Value> &)> matches);
void _deleteMatchingEntriesAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches);
bool _deleteMatchingEntryAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches);
mutable std::mutex _mutex;
cpputils::LockPool<Key> _currentlyFlushingEntries;
QueueMap<Key, CacheEntry<Key, Value>> _cachedBlocks;
std::unique_ptr<PeriodicTask> _timeoutFlusher;
DISALLOW_COPY_AND_ASSIGN(Cache);
};
template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Key, Value, MAX_ENTRIES>::PURGE_LIFETIME_SEC;
template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Key, Value, MAX_ENTRIES>::PURGE_INTERVAL;
template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Key, Value, MAX_ENTRIES>::MAX_LIFETIME_SEC;
template<class Key, class Value, uint32_t MAX_ENTRIES>
Cache<Key, Value, MAX_ENTRIES>::Cache(): _mutex(), _currentlyFlushingEntries(), _cachedBlocks(), _timeoutFlusher(nullptr) {
//Don't initialize timeoutFlusher in the initializer list,
//because it then might already call Cache::popOldEntries() before Cache is done constructing.
_timeoutFlusher = std::make_unique<PeriodicTask>(std::bind(&Cache::_deleteOldEntriesParallel, this), PURGE_INTERVAL);
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
Cache<Key, Value, MAX_ENTRIES>::~Cache() {
_deleteAllEntriesParallel();
ASSERT(_cachedBlocks.size() == 0, "Error in _deleteAllEntriesParallel()");
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
boost::optional<Value> Cache<Key, Value, MAX_ENTRIES>::pop(const Key &key) {
std::unique_lock<std::mutex> lock(_mutex);
cpputils::MutexPoolLock<Key> lockEntryFromBeingPopped(&_currentlyFlushingEntries, key, &lock);
auto found = _cachedBlocks.pop(key);
if (!found) {
return boost::none;
}
return found->releaseValue();
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::push(const Key &key, Value value) {
std::unique_lock<std::mutex> lock(_mutex);
ASSERT(_cachedBlocks.size() <= MAX_ENTRIES, "Cache too full");
_makeSpaceForEntry(&lock);
_cachedBlocks.push(key, CacheEntry<Key, Value>(std::move(value)));
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_makeSpaceForEntry(std::unique_lock<std::mutex> *lock) {
// _deleteEntry releases the lock while the Value destructor is running.
// So we can destruct multiple entries in parallel and also call pop() or push() while doing so.
// However, if another thread calls push() before we get the lock back, the cache is full again.
// That's why we need the while() loop here.
while (_cachedBlocks.size() == MAX_ENTRIES) {
_deleteEntry(lock);
}
ASSERT(_cachedBlocks.size() < MAX_ENTRIES, "Removing entry from cache didn't work");
};
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_deleteEntry(std::unique_lock<std::mutex> *lock) {
auto key = _cachedBlocks.peekKey();
ASSERT(key != boost::none, "There was no entry to delete");
cpputils::MutexPoolLock<Key> lockEntryFromBeingPopped(&_currentlyFlushingEntries, *key);
auto value = _cachedBlocks.pop();
// Call destructor outside of the unique_lock,
// i.e. pop() and push() can be called here, except for pop() on the element in _currentlyFlushingEntries
lock->unlock();
value = boost::none; // Call destructor
lock->lock();
};
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_deleteAllEntriesParallel() {
return _deleteMatchingEntriesAtBeginningParallel([] (const CacheEntry<Key, Value> &) {
return true;
});
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_deleteOldEntriesParallel() {
return _deleteMatchingEntriesAtBeginningParallel([] (const CacheEntry<Key, Value> &entry) {
return entry.ageSeconds() > PURGE_LIFETIME_SEC;
});
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_deleteMatchingEntriesAtBeginningParallel(std::function<bool (const CacheEntry<Key, Value> &)> matches) {
// Twice the number of cores, so we use full CPU even if half the threads are doing I/O
unsigned int numThreads = 2 * std::max(1u, std::thread::hardware_concurrency());
std::vector<std::future<void>> waitHandles;
for (unsigned int i = 0; i < numThreads; ++i) {
waitHandles.push_back(std::async(std::launch::async, [this, matches] {
_deleteMatchingEntriesAtBeginning(matches);
}));
}
for (auto & waitHandle : waitHandles) {
waitHandle.wait();
}
};
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_deleteMatchingEntriesAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches) {
while (_deleteMatchingEntryAtBeginning(matches)) {}
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
bool Cache<Key, Value, MAX_ENTRIES>::_deleteMatchingEntryAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches) {
// This function can be called in parallel by multiple threads and will then cause the Value destructors
// to be called in parallel. The call to _deleteEntry() releases the lock while the Value destructor is running.
std::unique_lock<std::mutex> lock(_mutex);
if (_cachedBlocks.size() > 0 && matches(*_cachedBlocks.peek())) {
_deleteEntry(&lock);
return true;
} else {
return false;
}
};
template<class Key, class Value, uint32_t MAX_ENTRIES>
uint32_t Cache<Key, Value, MAX_ENTRIES>::size() const {
std::unique_lock<std::mutex> lock(_mutex);
return _cachedBlocks.size();
};
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::flush() {
//TODO Test flush()
return _deleteAllEntriesParallel();
};
}
}
#endif

1
src/blockstore/implementations/caching/cache/CacheEntry.cpp vendored Normal file
View File

@ -0,0 +1 @@
#include "CacheEntry.h"

44
src/blockstore/implementations/caching/cache/CacheEntry.h vendored Normal file
View File

@ -0,0 +1,44 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHEENTRY_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHEENTRY_H_
#include <ctime>
#include <memory>
#include <cpp-utils/macros.h>
#include <boost/date_time/posix_time/posix_time_types.hpp>
namespace blockstore {
namespace caching {
template<class Key, class Value>
class CacheEntry final {
public:
explicit CacheEntry(Value value): _lastAccess(currentTime()), _value(std::move(value)) {
}
CacheEntry(CacheEntry &&) = default;
double ageSeconds() const {
return ((double)(currentTime() - _lastAccess).total_nanoseconds()) / ((double)1000000000);
}
Value releaseValue() {
return std::move(_value);
}
private:
boost::posix_time::ptime _lastAccess;
Value _value;
static boost::posix_time::ptime currentTime() {
return boost::posix_time::microsec_clock::local_time();
}
DISALLOW_COPY_AND_ASSIGN(CacheEntry);
};
}
}
#endif

27
src/blockstore/implementations/caching/cache/PeriodicTask.cpp vendored Normal file
View File

@ -0,0 +1,27 @@
#include "PeriodicTask.h"
#include <cpp-utils/logging/logging.h>
using std::function;
using std::endl;
using namespace cpputils::logging;
namespace blockstore {
namespace caching {
PeriodicTask::PeriodicTask(function<void ()> task, double intervalSec) :
_task(task),
_interval((uint64_t)(UINT64_C(1000000000) * intervalSec)),
_thread(std::bind(&PeriodicTask::_loopIteration, this)) {
_thread.start();
}
bool PeriodicTask::_loopIteration() {
//Has to be boost::this_thread::sleep_for and not std::this_thread::sleep_for, because it has to be interruptible.
//LoopThread will interrupt this method if it has to be restarted.
boost::this_thread::sleep_for(_interval);
_task();
return true; // Run another iteration (don't terminate thread)
}
}
}

32
src/blockstore/implementations/caching/cache/PeriodicTask.h vendored Normal file
View File

@ -0,0 +1,32 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_PERIODICTASK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_PERIODICTASK_H_
#include <functional>
#include <cpp-utils/thread/LoopThread.h>
#include <boost/chrono.hpp>
namespace blockstore {
namespace caching {
class PeriodicTask final {
public:
PeriodicTask(std::function<void ()> task, double intervalSec);
private:
bool _loopIteration();
std::function<void()> _task;
boost::chrono::nanoseconds _interval;
//This member has to be last, so the thread is destructed first. Otherwise the thread might access elements from a
//partly destructed PeriodicTask.
cpputils::LoopThread _thread;
DISALLOW_COPY_AND_ASSIGN(PeriodicTask);
};
}
}
#endif

1
src/blockstore/implementations/caching/cache/QueueMap.cpp vendored Normal file
View File

@ -0,0 +1 @@
#include "QueueMap.h"

121
src/blockstore/implementations/caching/cache/QueueMap.h vendored Normal file
View File

@ -0,0 +1,121 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_QUEUEMAP_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_QUEUEMAP_H_
#include <memory>
#include <unordered_map>
#include <cassert>
#include <boost/optional.hpp>
#include <cpp-utils/macros.h>
#include <cpp-utils/assert/assert.h>
namespace blockstore {
namespace caching {
//TODO FreeList for performance (malloc is expensive)
//TODO Single linked list with pointer to last element (for insertion) should be enough for a queue. No double linked list needed.
// But then, popping arbitrary elements needs to be rewritten so that _removeFromQueue() is _removeSuccessorFromQueue()
// and the map doesn't store the element itself, but its predecessor. That is, popping might be a bit slower. Test with experiments!
// A class that is a queue and a map at the same time. We could also see it as an addressable queue.
template<class Key, class Value>
class QueueMap final {
public:
QueueMap(): _entries(), _sentinel(&_sentinel, &_sentinel) {
}
~QueueMap() {
for (auto &entry : _entries) {
entry.second.release();
}
}
void push(const Key &key, Value value) {
auto newEntry = _entries.emplace(std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple(_sentinel.prev, &_sentinel));
if (!newEntry.second) {
throw std::logic_error("There is already an element with this key");
}
newEntry.first->second.init(&newEntry.first->first, std::move(value));
//The following is ok, because std::unordered_map never invalidates pointers to its entries
_sentinel.prev->next = &newEntry.first->second;
_sentinel.prev = &newEntry.first->second;
}
boost::optional<Value> pop(const Key &key) {
auto found = _entries.find(key);
if (found == _entries.end()) {
return boost::none;
}
_removeFromQueue(found->second);
auto value = found->second.release();
_entries.erase(found);
return std::move(value);
}
boost::optional<Value> pop() {
if(_sentinel.next == &_sentinel) {
return boost::none;
}
return pop(*_sentinel.next->key);
}
boost::optional<const Key &> peekKey() {
if(_sentinel.next == &_sentinel) {
return boost::none;
}
return *_sentinel.next->key;
}
boost::optional<const Value &> peek() {
if(_sentinel.next == &_sentinel) {
return boost::none;
}
return _sentinel.next->value();
}
uint32_t size() const {
return _entries.size();
}
private:
class Entry final {
public:
Entry(Entry *prev_, Entry *next_): prev(prev_), next(next_), key(nullptr), __value() {
}
void init(const Key *key_, Value value_) {
key = key_;
new(__value) Value(std::move(value_));
}
Value release() {
Value value = std::move(*_value());
_value()->~Value();
return value;
}
const Value &value() {
return *_value();
}
Entry *prev;
Entry *next;
const Key *key;
private:
Value *_value() {
return reinterpret_cast<Value*>(__value);
}
alignas(Value) char __value[sizeof(Value)];
DISALLOW_COPY_AND_ASSIGN(Entry);
};
void _removeFromQueue(const Entry &entry) {
entry.prev->next = entry.next;
entry.next->prev = entry.prev;
}
std::unordered_map<Key, Entry> _entries;
Entry _sentinel;
DISALLOW_COPY_AND_ASSIGN(QueueMap);
};
}
}
#endif

1
src/blockstore/implementations/compressing/CompressedBlock.cpp Normal file
View File

@ -0,0 +1 @@
#include "CompressedBlock.h"

127
src/blockstore/implementations/compressing/CompressedBlock.h Normal file
View File

@ -0,0 +1,127 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSEDBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSEDBLOCK_H_
#include "../../interface/Block.h"
#include "../../interface/BlockStore.h"
#include <cpp-utils/data/DataUtils.h>
#include <cpp-utils/pointer/unique_ref.h>
#include <mutex>
namespace blockstore {
class BlockStore;
namespace compressing {
template<class Compressor> class CompressingBlockStore;
template<class Compressor>
class CompressedBlock final: public Block {
public:
static boost::optional<cpputils::unique_ref<CompressedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData);
static cpputils::unique_ref<CompressedBlock> Decompress(cpputils::unique_ref<Block> baseBlock);
CompressedBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data decompressedData);
~CompressedBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref<Block> releaseBaseBlock();
private:
void _compressToBaseBlock();
cpputils::unique_ref<Block> _baseBlock;
cpputils::Data _decompressedData;
std::mutex _mutex;
bool _dataChanged;
DISALLOW_COPY_AND_ASSIGN(CompressedBlock);
};
template<class Compressor>
boost::optional<cpputils::unique_ref<CompressedBlock<Compressor>>> CompressedBlock<Compressor>::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData) {
cpputils::Data compressed = Compressor::Compress(decompressedData);
auto baseBlock = baseBlockStore->tryCreate(key, std::move(compressed));
if (baseBlock == boost::none) {
//TODO Test this code branch
return boost::none;
}
return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(*baseBlock), std::move(decompressedData));
}
template<class Compressor>
cpputils::unique_ref<CompressedBlock<Compressor>> CompressedBlock<Compressor>::Decompress(cpputils::unique_ref<Block> baseBlock) {
cpputils::Data decompressed = Compressor::Decompress((byte*)baseBlock->data(), baseBlock->size());
return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(baseBlock), std::move(decompressed));
}
template<class Compressor>
CompressedBlock<Compressor>::CompressedBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data decompressedData)
: Block(baseBlock->key()),
_baseBlock(std::move(baseBlock)),
_decompressedData(std::move(decompressedData)),
_dataChanged(false) {
}
template<class Compressor>
CompressedBlock<Compressor>::~CompressedBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_compressToBaseBlock();
}
template<class Compressor>
const void *CompressedBlock<Compressor>::data() const {
return _decompressedData.data();
}
template<class Compressor>
void CompressedBlock<Compressor>::write(const void *source, uint64_t offset, uint64_t size) {
std::memcpy((uint8_t*)_decompressedData.dataOffset(offset), source, size);
_dataChanged = true;
}
template<class Compressor>
void CompressedBlock<Compressor>::flush() {
std::unique_lock<std::mutex> lock(_mutex);
_compressToBaseBlock();
return _baseBlock->flush();
}
template<class Compressor>
size_t CompressedBlock<Compressor>::size() const {
return _decompressedData.size();
}
template<class Compressor>
void CompressedBlock<Compressor>::resize(size_t newSize) {
_decompressedData = cpputils::DataUtils::resize(std::move(_decompressedData), newSize);
_dataChanged = true;
}
template<class Compressor>
cpputils::unique_ref<Block> CompressedBlock<Compressor>::releaseBaseBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_compressToBaseBlock();
return std::move(_baseBlock);
}
template<class Compressor>
void CompressedBlock<Compressor>::_compressToBaseBlock() {
if (_dataChanged) {
cpputils::Data compressed = Compressor::Compress(_decompressedData);
_baseBlock->resize(compressed.size());
_baseBlock->write(compressed.data(), 0, compressed.size());
_dataChanged = false;
}
}
}
}
#endif

1
src/blockstore/implementations/compressing/CompressingBlockStore.cpp Normal file
View File

@ -0,0 +1 @@
#include "CompressingBlockStore.h"

77
src/blockstore/implementations/compressing/CompressingBlockStore.h Normal file
View File

@ -0,0 +1,77 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSINGBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSINGBLOCKSTORE_H_
#include "../../interface/BlockStore.h"
#include "CompressedBlock.h"
namespace blockstore {
namespace compressing {
template<class Compressor>
class CompressingBlockStore final: public BlockStore {
public:
CompressingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
~CompressingBlockStore();
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
DISALLOW_COPY_AND_ASSIGN(CompressingBlockStore);
};
template<class Compressor>
CompressingBlockStore<Compressor>::CompressingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore)
: _baseBlockStore(std::move(baseBlockStore)) {
}
template<class Compressor>
CompressingBlockStore<Compressor>::~CompressingBlockStore() {
}
template<class Compressor>
Key CompressingBlockStore<Compressor>::createKey() {
return _baseBlockStore->createKey();
}
template<class Compressor>
boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::tryCreate(const Key &key, cpputils::Data data) {
auto result = CompressedBlock<Compressor>::TryCreateNew(_baseBlockStore.get(), key, std::move(data));
if (result == boost::none) {
return boost::none;
}
return cpputils::unique_ref<Block>(std::move(*result));
}
template<class Compressor>
boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::load(const Key &key) {
auto loaded = _baseBlockStore->load(key);
if (loaded == boost::none) {
return boost::none;
}
return boost::optional<cpputils::unique_ref<Block>>(CompressedBlock<Compressor>::Decompress(std::move(*loaded)));
}
template<class Compressor>
void CompressingBlockStore<Compressor>::remove(cpputils::unique_ref<Block> block) {
auto _block = cpputils::dynamic_pointer_move<CompressedBlock<Compressor>>(block);
ASSERT(_block != boost::none, "Wrong block type");
auto baseBlock = (*_block)->releaseBaseBlock();
return _baseBlockStore->remove(std::move(baseBlock));
}
template<class Compressor>
uint64_t CompressingBlockStore<Compressor>::numBlocks() const {
return _baseBlockStore->numBlocks();
}
}
}
#endif

29
src/blockstore/implementations/compressing/compressors/Gzip.cpp Normal file
View File

@ -0,0 +1,29 @@
#include "Gzip.h"
#include <cryptopp/gzip.h>
using cpputils::Data;
namespace blockstore {
namespace compressing {
Data Gzip::Compress(const Data &data) {
CryptoPP::Gzip zipper;
zipper.Put((byte *) data.data(), data.size());
zipper.MessageEnd();
Data compressed(zipper.MaxRetrievable());
zipper.Get((byte *) compressed.data(), compressed.size());
return compressed;
}
Data Gzip::Decompress(const void *data, size_t size) {
//TODO Change interface to taking cpputils::Data objects (needs changing blockstore so we can read their "class Data", because this is called from CompressedBlock::Decompress()).
CryptoPP::Gunzip zipper;
zipper.Put((byte *) data, size);
zipper.MessageEnd();
Data decompressed(zipper.MaxRetrievable());
zipper.Get((byte *) decompressed.data(), decompressed.size());
return decompressed;
}
}
}

18
src/blockstore/implementations/compressing/compressors/Gzip.h Normal file
View File

@ -0,0 +1,18 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_GZIP_H
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_GZIP_H
#include <cpp-utils/data/Data.h>
namespace blockstore {
namespace compressing {
class Gzip {
public:
static cpputils::Data Compress(const cpputils::Data &data);
static cpputils::Data Decompress(const void *data, size_t size);
};
}
}
#endif

139
src/blockstore/implementations/compressing/compressors/RunLengthEncoding.cpp Normal file
View File

@ -0,0 +1,139 @@
#include "RunLengthEncoding.h"
#include <sstream>
#include <cpp-utils/assert/assert.h>
using cpputils::Data;
using std::string;
using std::ostringstream;
using std::istringstream;
namespace blockstore {
namespace compressing {
// Alternatively store a run of arbitrary bytes and a run of identical bytes.
// Each run is preceded by its length. Length fields are uint16_t.
// Example: 2 - 5 - 8 - 10 - 3 - 0 - 2 - 0
// Length 2 arbitrary bytes (values: 5, 8), the next 10 bytes store "3" each,
// then 0 arbitrary bytes and 2x "0".
Data RunLengthEncoding::Compress(const Data &data) {
ostringstream compressed;
uint8_t *current = (uint8_t*)data.data();
uint8_t *end = (uint8_t*)data.data()+data.size();
while (current < end) {
_encodeArbitraryWords(&current, end, &compressed);
ASSERT(current <= end, "Overflow");
if (current == end) {
break;
}
_encodeIdenticalWords(&current, end, &compressed);
ASSERT(current <= end, "Overflow");
}
return _extractData(&compressed);
}
void RunLengthEncoding::_encodeArbitraryWords(uint8_t **current, uint8_t* end, ostringstream *output) {
uint16_t size = _arbitraryRunLength(*current, end);
output->write((const char*)&size, sizeof(uint16_t));
output->write((const char*)*current, size);
*current += size;
}
uint16_t RunLengthEncoding::_arbitraryRunLength(uint8_t *start, uint8_t* end) {
// Each stopping of an arbitrary bytes run costs us 5 byte, because we have to store the length
// for the identical bytes run (2 byte), the identical byte itself (1 byte) and the length for the next arbitrary bytes run (2 byte).
// So to get an advantage from stopping an arbitrary bytes run, at least 6 bytes have to be identical.
// realEnd avoids an overflow of the 16bit counter
uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max());
// Count the number of identical bytes and return if it finds a run of more than 6 identical bytes.
uint8_t lastByte = *start + 1; // Something different from the first byte
uint8_t numIdenticalBytes = 1;
for(uint8_t *current = start; current != realEnd; ++current) {
if (*current == lastByte) {
++numIdenticalBytes;
if (numIdenticalBytes == 6) {
return current - start - 5; //-5, because the end pointer for the arbitrary byte run should point to the first identical byte, not the one before.
}
} else {
numIdenticalBytes = 1;
}
lastByte = *current;
}
//It wasn't worth stopping the arbitrary bytes run anywhere. The whole region should be an arbitrary run.
return realEnd-start;
}
void RunLengthEncoding::_encodeIdenticalWords(uint8_t **current, uint8_t* end, ostringstream *output) {
uint16_t size = _countIdenticalBytes(*current, end);
output->write((const char*)&size, sizeof(uint16_t));
output->write((const char*)*current, 1);
*current += size;
}
uint16_t RunLengthEncoding::_countIdenticalBytes(uint8_t *start, uint8_t *end) {
uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max()); // This prevents overflow of the 16bit counter
for (uint8_t *current = start+1; current != realEnd; ++current) {
if (*current != *start) {
return current-start;
}
}
// All bytes have been identical
return realEnd - start;
}
Data RunLengthEncoding::_extractData(ostringstream *stream) {
string str = stream->str();
Data data(str.size());
std::memcpy(data.data(), str.c_str(), str.size());
return data;
}
Data RunLengthEncoding::Decompress(const void *data, size_t size) {
istringstream stream;
_parseData((uint8_t*)data, size, &stream);
ostringstream decompressed;
while(_hasData(&stream)) {
_decodeArbitraryWords(&stream, &decompressed);
if (!_hasData(&stream)) {
break;
}
_decodeIdenticalWords(&stream, &decompressed);
}
return _extractData(&decompressed);
}
bool RunLengthEncoding::_hasData(istringstream *str) {
str->peek();
return !str->eof();
}
void RunLengthEncoding::_parseData(const uint8_t *data, size_t size, istringstream *result) {
result->str(string((const char*)data, size));
}
void RunLengthEncoding::_decodeArbitraryWords(istringstream *stream, ostringstream *decompressed) {
uint16_t size;
stream->read((char*)&size, sizeof(uint16_t));
ASSERT(stream->good(), "Premature end of stream");
Data run(size);
stream->read((char*)run.data(), size);
ASSERT(stream->good(), "Premature end of stream");
decompressed->write((const char*)run.data(), run.size());
}
void RunLengthEncoding::_decodeIdenticalWords(istringstream *stream, ostringstream *decompressed) {
uint16_t size;
stream->read((char*)&size, sizeof(uint16_t));
ASSERT(stream->good(), "Premature end of stream");
uint8_t value;
stream->read((char*)&value, 1);
ASSERT(stream->good(), "Premature end of stream");
Data run(size);
std::memset(run.data(), value, run.size());
decompressed->write((const char*)run.data(), run.size());
}
}
}

29
src/blockstore/implementations/compressing/compressors/RunLengthEncoding.h Normal file
View File

@ -0,0 +1,29 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_RUNLENGTHENCODING_H
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_RUNLENGTHENCODING_H
#include <cpp-utils/data/Data.h>
namespace blockstore {
namespace compressing {
class RunLengthEncoding {
public:
static cpputils::Data Compress(const cpputils::Data &data);
static cpputils::Data Decompress(const void *data, size_t size);
private:
static void _encodeArbitraryWords(uint8_t **current, uint8_t* end, std::ostringstream *output);
static uint16_t _arbitraryRunLength(uint8_t *start, uint8_t* end);
static void _encodeIdenticalWords(uint8_t **current, uint8_t* end, std::ostringstream *output);
static uint16_t _countIdenticalBytes(uint8_t *start, uint8_t *end);
static bool _hasData(std::istringstream *stream);
static cpputils::Data _extractData(std::ostringstream *stream);
static void _parseData(const uint8_t *data, size_t size, std::istringstream *result);
static void _decodeArbitraryWords(std::istringstream *stream, std::ostringstream *decompressed);
static void _decodeIdenticalWords(std::istringstream *stream, std::ostringstream *decompressed);
};
}
}
#endif

1
src/blockstore/implementations/encrypted/EncryptedBlock.cpp Normal file
View File

@ -0,0 +1 @@
#include "EncryptedBlock.h"

177
src/blockstore/implementations/encrypted/EncryptedBlock.h Normal file
View File

@ -0,0 +1,177 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include "../../interface/BlockStore.h"
#include <cpp-utils/macros.h>
#include <memory>
#include <iostream>
#include <boost/optional.hpp>
#include <cpp-utils/crypto/symmetric/Cipher.h>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
#include <mutex>
#include <cpp-utils/logging/logging.h>
namespace blockstore {
namespace encrypted {
template<class Cipher> class EncryptedBlockStore;
//TODO Test EncryptedBlock
//TODO Fix mutexes & locks (basically true for all blockstores)
template<class Cipher>
class EncryptedBlock final: public Block {
public:
BOOST_CONCEPT_ASSERT((cpputils::CipherConcept<Cipher>));
static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey);
static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key);
//TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key, cpputils::Data plaintextWithHeader);
~EncryptedBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t count) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref<Block> releaseBlock();
private:
cpputils::unique_ref<Block> _baseBlock; // TODO Do I need the ciphertext block in memory or is the key enough?
cpputils::Data _plaintextWithHeader;
typename Cipher::EncryptionKey _encKey;
bool _dataChanged;
static constexpr unsigned int HEADER_LENGTH = Key::BINARY_LENGTH;
void _encryptToBaseBlock();
static cpputils::Data _prependKeyHeaderToData(const Key &key, cpputils::Data data);
static bool _keyHeaderIsCorrect(const Key &key, const cpputils::Data &data);
std::mutex _mutex;
DISALLOW_COPY_AND_ASSIGN(EncryptedBlock);
};
template<class Cipher>
constexpr unsigned int EncryptedBlock<Cipher>::HEADER_LENGTH;
template<class Cipher>
boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey) {
cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, std::move(data));
cpputils::Data encrypted = Cipher::encrypt((byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), encKey);
auto baseBlock = baseBlockStore->tryCreate(key, std::move(encrypted));
if (baseBlock == boost::none) {
//TODO Test this code branch
return boost::none;
}
return cpputils::make_unique_ref<EncryptedBlock>(std::move(*baseBlock), encKey, std::move(plaintextWithHeader));
}
template<class Cipher>
boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey) {
//TODO Change BlockStore so we can read their "class Data" objects instead of "void *data()", and then we can change the Cipher interface to take Data objects instead of "byte *" + size
boost::optional<cpputils::Data> plaintextWithHeader = Cipher::decrypt((byte*)baseBlock->data(), baseBlock->size(), encKey);
if(plaintextWithHeader == boost::none) {
//Decryption failed (e.g. an authenticated cipher detected modifications to the ciphertext)
cpputils::logging::LOG(cpputils::logging::WARN) << "Decrypting block " << baseBlock->key().ToString() << " failed. Was the block modified by an attacker?";
return boost::none;
}
if(!_keyHeaderIsCorrect(baseBlock->key(), *plaintextWithHeader)) {
//The stored key in the block data is incorrect - an attacker might have exchanged the contents with the encrypted data from a different block
cpputils::logging::LOG(cpputils::logging::WARN) << "Decrypting block " << baseBlock->key().ToString() << " failed due to invalid block key. Was the block modified by an attacker?";
return boost::none;
}
return cpputils::make_unique_ref<EncryptedBlock<Cipher>>(std::move(baseBlock), encKey, std::move(*plaintextWithHeader));
}
template<class Cipher>
cpputils::Data EncryptedBlock<Cipher>::_prependKeyHeaderToData(const Key &key, cpputils::Data data) {
static_assert(HEADER_LENGTH >= Key::BINARY_LENGTH, "Key doesn't fit into the header");
cpputils::Data result(data.size() + HEADER_LENGTH);
std::memcpy(result.data(), key.data(), Key::BINARY_LENGTH);
std::memcpy((uint8_t*)result.data() + Key::BINARY_LENGTH, data.data(), data.size());
return result;
}
template<class Cipher>
bool EncryptedBlock<Cipher>::_keyHeaderIsCorrect(const Key &key, const cpputils::Data &data) {
return 0 == std::memcmp(key.data(), data.data(), Key::BINARY_LENGTH);
}
template<class Cipher>
EncryptedBlock<Cipher>::EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey, cpputils::Data plaintextWithHeader)
:Block(baseBlock->key()),
_baseBlock(std::move(baseBlock)),
_plaintextWithHeader(std::move(plaintextWithHeader)),
_encKey(encKey),
_dataChanged(false),
_mutex() {
}
template<class Cipher>
EncryptedBlock<Cipher>::~EncryptedBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
}
template<class Cipher>
const void *EncryptedBlock<Cipher>::data() const {
return (uint8_t*)_plaintextWithHeader.data() + HEADER_LENGTH;
}
template<class Cipher>
void EncryptedBlock<Cipher>::write(const void *source, uint64_t offset, uint64_t count) {
ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition
std::memcpy((uint8_t*)_plaintextWithHeader.data()+HEADER_LENGTH+offset, source, count);
_dataChanged = true;
}
template<class Cipher>
void EncryptedBlock<Cipher>::flush() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
return _baseBlock->flush();
}
template<class Cipher>
size_t EncryptedBlock<Cipher>::size() const {
return _plaintextWithHeader.size() - HEADER_LENGTH;
}
template<class Cipher>
void EncryptedBlock<Cipher>::resize(size_t newSize) {
_plaintextWithHeader = cpputils::DataUtils::resize(std::move(_plaintextWithHeader), newSize + HEADER_LENGTH);
_dataChanged = true;
}
template<class Cipher>
void EncryptedBlock<Cipher>::_encryptToBaseBlock() {
if (_dataChanged) {
cpputils::Data encrypted = Cipher::encrypt((byte*)_plaintextWithHeader.data(), _plaintextWithHeader.size(), _encKey);
_baseBlock->write(encrypted.data(), 0, encrypted.size());
_dataChanged = false;
}
}
template<class Cipher>
cpputils::unique_ref<Block> EncryptedBlock<Cipher>::releaseBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
return std::move(_baseBlock);
}
}
}
#endif

1
src/blockstore/implementations/encrypted/EncryptedBlockStore.cpp Normal file
View File

@ -0,0 +1 @@
#include "EncryptedBlockStore.h"

90
src/blockstore/implementations/encrypted/EncryptedBlockStore.h Normal file
View File

@ -0,0 +1,90 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
#include "../../interface/BlockStore.h"
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/cast.h>
#include "EncryptedBlock.h"
#include <iostream>
namespace blockstore {
namespace encrypted {
template<class Cipher>
class EncryptedBlockStore final: public BlockStore {
public:
EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey);
//TODO Are createKey() tests included in generic BlockStoreTest? If not, add it!
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
//This function should only be used by test cases
void __setKey(const typename Cipher::EncryptionKey &encKey);
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
typename Cipher::EncryptionKey _encKey;
DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore);
};
template<class Cipher>
EncryptedBlockStore<Cipher>::EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
: _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
}
template<class Cipher>
Key EncryptedBlockStore<Cipher>::createKey() {
return _baseBlockStore->createKey();
}
template<class Cipher>
boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::tryCreate(const Key &key, cpputils::Data data) {
//TODO Test that this returns boost::none when base blockstore returns nullptr (for all pass-through-blockstores)
//TODO Easier implementation? This is only so complicated because of the case EncryptedBlock -> Block
auto result = EncryptedBlock<Cipher>::TryCreateNew(_baseBlockStore.get(), key, std::move(data), _encKey);
if (result == boost::none) {
return boost::none;
}
return cpputils::unique_ref<Block>(std::move(*result));
}
template<class Cipher>
boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::load(const Key &key) {
auto block = _baseBlockStore->load(key);
if (block == boost::none) {
//TODO Test this path (for all pass-through-blockstores)
return boost::none;
}
return boost::optional<cpputils::unique_ref<Block>>(EncryptedBlock<Cipher>::TryDecrypt(std::move(*block), _encKey));
}
template<class Cipher>
void EncryptedBlockStore<Cipher>::remove(cpputils::unique_ref<Block> block) {
auto encryptedBlock = cpputils::dynamic_pointer_move<EncryptedBlock<Cipher>>(block);
ASSERT(encryptedBlock != boost::none, "Block is not an EncryptedBlock");
auto baseBlock = (*encryptedBlock)->releaseBlock();
return _baseBlockStore->remove(std::move(baseBlock));
}
template<class Cipher>
uint64_t EncryptedBlockStore<Cipher>::numBlocks() const {
return _baseBlockStore->numBlocks();
}
template<class Cipher>
void EncryptedBlockStore<Cipher>::__setKey(const typename Cipher::EncryptionKey &encKey) {
_encKey = encKey;
}
}
}
#endif

50
src/blockstore/implementations/inmemory/InMemoryBlock.cpp Normal file
View File

@ -0,0 +1,50 @@
#include "InMemoryBlock.h"
#include "InMemoryBlockStore.h"
#include <cstring>
#include <cpp-utils/data/DataUtils.h>
#include <cpp-utils/assert/assert.h>
using std::make_shared;
using std::istream;
using std::ostream;
using std::ifstream;
using std::ofstream;
using std::ios;
using cpputils::Data;
namespace blockstore {
namespace inmemory {
InMemoryBlock::InMemoryBlock(const Key &key, Data data)
: Block(key), _data(make_shared<Data>(std::move(data))) {
}
InMemoryBlock::InMemoryBlock(const InMemoryBlock &rhs)
: Block(rhs), _data(rhs._data) {
}
InMemoryBlock::~InMemoryBlock() {
}
const void *InMemoryBlock::data() const {
return _data->data();
}
void InMemoryBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data->size() && offset + size <= _data->size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
std::memcpy((uint8_t*)_data->data()+offset, source, size);
}
size_t InMemoryBlock::size() const {
return _data->size();
}
void InMemoryBlock::resize(size_t newSize) {
*_data = cpputils::DataUtils::resize(std::move(*_data), newSize);
}
void InMemoryBlock::flush() {
}
}
}

33
src/blockstore/implementations/inmemory/InMemoryBlock.h Normal file
View File

@ -0,0 +1,33 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
namespace blockstore {
namespace inmemory {
class InMemoryBlockStore;
class InMemoryBlock final: public Block {
public:
InMemoryBlock(const Key &key, cpputils::Data size);
InMemoryBlock(const InMemoryBlock &rhs);
~InMemoryBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
private:
std::shared_ptr<cpputils::Data> _data;
};
}
}
#endif

56
src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp Normal file
View File

@ -0,0 +1,56 @@
#include "InMemoryBlock.h"
#include "InMemoryBlockStore.h"
#include <memory>
#include <cpp-utils/assert/assert.h>
using std::make_unique;
using std::string;
using std::mutex;
using std::lock_guard;
using std::piecewise_construct;
using std::make_tuple;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
namespace blockstore {
namespace inmemory {
InMemoryBlockStore::InMemoryBlockStore()
: _blocks() {}
optional<unique_ref<Block>> InMemoryBlockStore::tryCreate(const Key &key, Data data) {
auto insert_result = _blocks.emplace(piecewise_construct, make_tuple(key.ToString()), make_tuple(key, std::move(data)));
if (!insert_result.second) {
return none;
}
//Return a pointer to the stored InMemoryBlock
return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(insert_result.first->second));
}
optional<unique_ref<Block>> InMemoryBlockStore::load(const Key &key) {
//Return a pointer to the stored InMemoryBlock
try {
return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(_blocks.at(key.ToString())));
} catch (const std::out_of_range &e) {
return none;
}
}
void InMemoryBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
cpputils::destruct(std::move(block));
int numRemoved = _blocks.erase(key.ToString());
ASSERT(1==numRemoved, "Didn't find block to remove");
}
uint64_t InMemoryBlockStore::numBlocks() const {
return _blocks.size();
}
}
}

33
src/blockstore/implementations/inmemory/InMemoryBlockStore.h Normal file
View File

@ -0,0 +1,33 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
#include <cpp-utils/macros.h>
#include <mutex>
#include <map>
namespace blockstore {
namespace inmemory {
class InMemoryBlock;
class InMemoryBlockStore final: public BlockStoreWithRandomKeys {
public:
InMemoryBlockStore();
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
private:
std::map<std::string, InMemoryBlock> _blocks;
DISALLOW_COPY_AND_ASSIGN(InMemoryBlockStore);
};
}
}
#endif

108
src/blockstore/implementations/ondisk/OnDiskBlock.cpp Normal file
View File

@ -0,0 +1,108 @@
#include <cstring>
#include <fstream>
#include <boost/filesystem.hpp>
#include "OnDiskBlock.h"
#include "OnDiskBlockStore.h"
#include "../../utils/FileDoesntExistException.h"
#include <cpp-utils/data/DataUtils.h>
#include <cpp-utils/assert/assert.h>
using std::istream;
using std::ostream;
using std::ifstream;
using std::ofstream;
using std::ios;
using cpputils::Data;
using cpputils::make_unique_ref;
using cpputils::unique_ref;
using boost::optional;
using boost::none;
namespace bf = boost::filesystem;
namespace blockstore {
namespace ondisk {
OnDiskBlock::OnDiskBlock(const Key &key, const bf::path &filepath, Data data)
: Block(key), _filepath(filepath), _data(std::move(data)), _dataChanged(false), _mutex() {
}
OnDiskBlock::~OnDiskBlock() {
flush();
}
const void *OnDiskBlock::data() const {
return _data.data();
}
void OnDiskBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
std::memcpy((uint8_t*)_data.data()+offset, source, size);
_dataChanged = true;
}
size_t OnDiskBlock::size() const {
return _data.size();
}
void OnDiskBlock::resize(size_t newSize) {
_data = cpputils::DataUtils::resize(std::move(_data), newSize);
_dataChanged = true;
}
optional<unique_ref<OnDiskBlock>> OnDiskBlock::LoadFromDisk(const bf::path &rootdir, const Key &key) {
auto filepath = rootdir / key.ToString();
try {
//If it isn't a file, Data::LoadFromFile() would usually also crash. We still need this extra check
//upfront, because Data::LoadFromFile() doesn't crash if we give it the path of a directory
//instead the path of a file.
//TODO Data::LoadFromFile now returns boost::optional. Do we then still need this?
if(!bf::is_regular_file(filepath)) {
return none;
}
boost::optional<Data> data = Data::LoadFromFile(filepath);
if (!data) {
return none;
}
return make_unique_ref<OnDiskBlock>(key, filepath, std::move(*data));
} catch (const FileDoesntExistException &e) {
return none;
}
}
optional<unique_ref<OnDiskBlock>> OnDiskBlock::CreateOnDisk(const bf::path &rootdir, const Key &key, Data data) {
auto filepath = rootdir / key.ToString();
if (bf::exists(filepath)) {
return none;
}
auto block = make_unique_ref<OnDiskBlock>(key, filepath, std::move(data));
block->_storeToDisk();
return std::move(block);
}
void OnDiskBlock::RemoveFromDisk(const bf::path &rootdir, const Key &key) {
auto filepath = rootdir / key.ToString();
ASSERT(bf::is_regular_file(filepath), "Block not found on disk");
bf::remove(filepath);
}
void OnDiskBlock::_fillDataWithZeroes() {
_data.FillWithZeroes();
_dataChanged = true;
}
void OnDiskBlock::_storeToDisk() const {
_data.StoreToFile(_filepath);
}
void OnDiskBlock::flush() {
std::unique_lock<std::mutex> lock(_mutex);
if (_dataChanged) {
_storeToDisk();
_dataChanged = false;
}
}
}
}

50
src/blockstore/implementations/ondisk/OnDiskBlock.h Normal file
View File

@ -0,0 +1,50 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
#include <boost/filesystem/path.hpp>
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include <iostream>
#include <cpp-utils/pointer/unique_ref.h>
#include <mutex>
namespace blockstore {
namespace ondisk {
class OnDiskBlockStore;
class OnDiskBlock final: public Block {
public:
OnDiskBlock(const Key &key, const boost::filesystem::path &filepath, cpputils::Data data);
~OnDiskBlock();
static boost::optional<cpputils::unique_ref<OnDiskBlock>> LoadFromDisk(const boost::filesystem::path &rootdir, const Key &key);
static boost::optional<cpputils::unique_ref<OnDiskBlock>> CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data);
static void RemoveFromDisk(const boost::filesystem::path &rootdir, const Key &key);
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
private:
const boost::filesystem::path _filepath;
cpputils::Data _data;
bool _dataChanged;
void _fillDataWithZeroes();
void _storeToDisk() const;
std::mutex _mutex;
DISALLOW_COPY_AND_ASSIGN(OnDiskBlock);
};
}
}
#endif

52
src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp Normal file
View File

@ -0,0 +1,52 @@
#include "OnDiskBlock.h"
#include "OnDiskBlockStore.h"
using std::string;
using cpputils::Data;
using cpputils::unique_ref;
using boost::optional;
using boost::none;
namespace bf = boost::filesystem;
namespace blockstore {
namespace ondisk {
OnDiskBlockStore::OnDiskBlockStore(const boost::filesystem::path &rootdir)
: _rootdir(rootdir) {
if (!bf::exists(rootdir)) {
throw std::runtime_error("Base directory not found");
}
if (!bf::is_directory(rootdir)) {
throw std::runtime_error("Base directory is not a directory");
}
//TODO Test for read access, write access, enter (x) access, and throw runtime_error in case
}
//TODO Do I have to lock tryCreate/remove and/or load? Or does ParallelAccessBlockStore take care of that?
optional<unique_ref<Block>> OnDiskBlockStore::tryCreate(const Key &key, Data data) {
//TODO Easier implementation? This is only so complicated because of the cast OnDiskBlock -> Block
auto result = std::move(OnDiskBlock::CreateOnDisk(_rootdir, key, std::move(data)));
if (result == boost::none) {
return boost::none;
}
return unique_ref<Block>(std::move(*result));
}
optional<unique_ref<Block>> OnDiskBlockStore::load(const Key &key) {
return optional<unique_ref<Block>>(OnDiskBlock::LoadFromDisk(_rootdir, key));
}
void OnDiskBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
cpputils::destruct(std::move(block));
OnDiskBlock::RemoveFromDisk(_rootdir, key);
}
uint64_t OnDiskBlockStore::numBlocks() const {
return std::distance(bf::directory_iterator(_rootdir), bf::directory_iterator());
}
}
}

31
src/blockstore/implementations/ondisk/OnDiskBlockStore.h Normal file
View File

@ -0,0 +1,31 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
#include <boost/filesystem.hpp>
#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
#include <cpp-utils/macros.h>
namespace blockstore {
namespace ondisk {
class OnDiskBlockStore final: public BlockStoreWithRandomKeys {
public:
explicit OnDiskBlockStore(const boost::filesystem::path &rootdir);
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
private:
const boost::filesystem::path _rootdir;
DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore);
};
}
}
#endif

1
src/blockstore/implementations/parallelaccess/BlockRef.cpp Normal file
View File

@ -0,0 +1 @@
#include "BlockRef.h"

48
src/blockstore/implementations/parallelaccess/BlockRef.h Normal file
View File

@ -0,0 +1,48 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_BLOCKREF_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_BLOCKREF_H_
#include <parallelaccessstore/ParallelAccessStore.h>
#include "../../interface/Block.h"
#include <cpp-utils/macros.h>
#include <memory>
namespace blockstore {
namespace parallelaccess {
class ParallelAccessBlockStore;
class BlockRef final: public Block, public parallelaccessstore::ParallelAccessStore<Block, BlockRef, Key>::ResourceRefBase {
public:
//TODO Unneccessarily storing Key twice here (in parent class and in _baseBlock).
explicit BlockRef(Block *baseBlock): Block(baseBlock->key()), _baseBlock(baseBlock) {}
const void *data() const override {
return _baseBlock->data();
}
void write(const void *source, uint64_t offset, uint64_t size) override {
return _baseBlock->write(source, offset, size);
}
void flush() override {
return _baseBlock->flush();
}
size_t size() const override {
return _baseBlock->size();
}
void resize(size_t newSize) override {
return _baseBlock->resize(newSize);
}
private:
Block *_baseBlock;
DISALLOW_COPY_AND_ASSIGN(BlockRef);
};
}
}
#endif

60
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.cpp Normal file
View File

@ -0,0 +1,60 @@
#include "BlockRef.h"
#include "ParallelAccessBlockStore.h"
#include "ParallelAccessBlockStoreAdapter.h"
#include <cassert>
#include <cpp-utils/pointer/cast.h>
#include <cpp-utils/assert/assert.h>
using std::string;
using std::promise;
using cpputils::dynamic_pointer_move;
using cpputils::make_unique_ref;
using boost::none;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
namespace blockstore {
namespace parallelaccess {
ParallelAccessBlockStore::ParallelAccessBlockStore(unique_ref<BlockStore> baseBlockStore)
: _baseBlockStore(std::move(baseBlockStore)), _parallelAccessStore(make_unique_ref<ParallelAccessBlockStoreAdapter>(_baseBlockStore.get())) {
}
Key ParallelAccessBlockStore::createKey() {
return _baseBlockStore->createKey();
}
optional<unique_ref<Block>> ParallelAccessBlockStore::tryCreate(const Key &key, cpputils::Data data) {
ASSERT(!_parallelAccessStore.isOpened(key), ("Key "+key.ToString()+"already exists").c_str());
auto block = _baseBlockStore->tryCreate(key, std::move(data));
if (block == none) {
//TODO Test this code branch
return none;
}
return unique_ref<Block>(_parallelAccessStore.add(key, std::move(*block)));
}
optional<unique_ref<Block>> ParallelAccessBlockStore::load(const Key &key) {
auto block = _parallelAccessStore.load(key);
if (block == none) {
return none;
}
return unique_ref<Block>(std::move(*block));
}
void ParallelAccessBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
auto block_ref = dynamic_pointer_move<BlockRef>(block);
ASSERT(block_ref != none, "Block is not a BlockRef");
return _parallelAccessStore.remove(key, std::move(*block_ref));
}
uint64_t ParallelAccessBlockStore::numBlocks() const {
return _baseBlockStore->numBlocks();
}
}
}

34
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h Normal file
View File

@ -0,0 +1,34 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTORE_H_
#include <parallelaccessstore/ParallelAccessStore.h>
#include "BlockRef.h"
#include "../../interface/BlockStore.h"
#include <cpp-utils/pointer/unique_ref.h>
namespace blockstore {
namespace parallelaccess {
//TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel)
class ParallelAccessBlockStore final: public BlockStore {
public:
explicit ParallelAccessBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
parallelaccessstore::ParallelAccessStore<Block, BlockRef, Key> _parallelAccessStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessBlockStore);
};
}
}
#endif

1
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp Normal file
View File

@ -0,0 +1 @@
#include "ParallelAccessBlockStoreAdapter.h"

35
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.h Normal file
View File

@ -0,0 +1,35 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTOREADAPTER_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTOREADAPTER_H_
#include <cpp-utils/macros.h>
#include <parallelaccessstore/ParallelAccessStore.h>
#include "../../interface/BlockStore.h"
namespace blockstore {
namespace parallelaccess {
class ParallelAccessBlockStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore<Block, Key> {
public:
explicit ParallelAccessBlockStoreAdapter(BlockStore *baseBlockStore)
:_baseBlockStore(std::move(baseBlockStore)) {
}
boost::optional<cpputils::unique_ref<Block>> loadFromBaseStore(const Key &key) override {
return _baseBlockStore->load(key);
}
void removeFromBaseStore(cpputils::unique_ref<Block> block) override {
return _baseBlockStore->remove(std::move(block));
}
private:
BlockStore *_baseBlockStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessBlockStoreAdapter);
};
}
}
#endif

54
src/blockstore/implementations/testfake/FakeBlock.cpp Normal file
View File

@ -0,0 +1,54 @@
#include "FakeBlock.h"
#include "FakeBlockStore.h"
#include <cstring>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
using std::shared_ptr;
using std::istream;
using std::ostream;
using std::ifstream;
using std::ofstream;
using std::ios;
using std::string;
using cpputils::Data;
namespace blockstore {
namespace testfake {
FakeBlock::FakeBlock(FakeBlockStore *store, const Key &key, shared_ptr<Data> data, bool dirty)
: Block(key), _store(store), _data(data), _dataChanged(dirty) {
}
FakeBlock::~FakeBlock() {
flush();
}
const void *FakeBlock::data() const {
return _data->data();
}
void FakeBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data->size() && offset + size <= _data->size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
std::memcpy((uint8_t*)_data->data()+offset, source, size);
_dataChanged = true;
}
size_t FakeBlock::size() const {
return _data->size();
}
void FakeBlock::resize(size_t newSize) {
*_data = cpputils::DataUtils::resize(std::move(*_data), newSize);
_dataChanged = true;
}
void FakeBlock::flush() {
if(_dataChanged) {
_store->updateData(key(), *_data);
_dataChanged = false;
}
}
}
}

39
src/blockstore/implementations/testfake/FakeBlock.h Normal file
View File

@ -0,0 +1,39 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include <cpp-utils/macros.h>
namespace blockstore {
namespace testfake {
class FakeBlockStore;
class FakeBlock final: public Block {
public:
FakeBlock(FakeBlockStore *store, const Key &key, std::shared_ptr<cpputils::Data> data, bool dirty);
~FakeBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
private:
FakeBlockStore *_store;
std::shared_ptr<cpputils::Data> _data;
bool _dataChanged;
DISALLOW_COPY_AND_ASSIGN(FakeBlock);
};
}
}
#endif

80
src/blockstore/implementations/testfake/FakeBlockStore.cpp Normal file
View File

@ -0,0 +1,80 @@
#include "FakeBlock.h"
#include "FakeBlockStore.h"
#include <cpp-utils/assert/assert.h>
using std::make_shared;
using std::string;
using std::mutex;
using std::lock_guard;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
namespace blockstore {
namespace testfake {
FakeBlockStore::FakeBlockStore()
: _blocks(), _used_dataregions_for_blocks(), _mutex() {}
optional<unique_ref<Block>> FakeBlockStore::tryCreate(const Key &key, Data data) {
std::unique_lock<std::mutex> lock(_mutex);
auto insert_result = _blocks.emplace(key.ToString(), std::move(data));
if (!insert_result.second) {
return none;
}
//Return a copy of the stored data
return _load(key);
}
optional<unique_ref<Block>> FakeBlockStore::load(const Key &key) {
std::unique_lock<std::mutex> lock(_mutex);
return _load(key);
}
optional<unique_ref<Block>> FakeBlockStore::_load(const Key &key) {
//Return a copy of the stored data
string key_string = key.ToString();
try {
return makeFakeBlockFromData(key, _blocks.at(key_string), false);
} catch (const std::out_of_range &e) {
return none;
}
}
void FakeBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
cpputils::destruct(std::move(block));
std::unique_lock<std::mutex> lock(_mutex);
int numRemoved = _blocks.erase(key.ToString());
ASSERT(numRemoved == 1, "Block not found");
}
unique_ref<Block> FakeBlockStore::makeFakeBlockFromData(const Key &key, const Data &data, bool dirty) {
auto newdata = make_shared<Data>(data.copy());
_used_dataregions_for_blocks.push_back(newdata);
return make_unique_ref<FakeBlock>(this, key, newdata, dirty);
}
void FakeBlockStore::updateData(const Key &key, const Data &data) {
std::unique_lock<std::mutex> lock(_mutex);
auto found = _blocks.find(key.ToString());
if (found == _blocks.end()) {
auto insertResult = _blocks.emplace(key.ToString(), data.copy());
ASSERT(true == insertResult.second, "Inserting didn't work");
found = insertResult.first;
}
Data &stored_data = found->second;
stored_data = data.copy();
}
uint64_t FakeBlockStore::numBlocks() const {
std::unique_lock<std::mutex> lock(_mutex);
return _blocks.size();
}
}
}

62
src/blockstore/implementations/testfake/FakeBlockStore.h Normal file
View File

@ -0,0 +1,62 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCKSTORE_H_
#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
#include <cpp-utils/data/Data.h>
#include <cpp-utils/macros.h>
#include <mutex>
#include <map>
namespace blockstore {
namespace testfake {
class FakeBlock;
/**
* This blockstore is meant to be used for unit tests when the module under test needs a blockstore to work with.
* It basically is the same as the InMemoryBlockStore, but much less forgiving for programming mistakes.
*
* InMemoryBlockStore for example simply ignores flushing and gives you access to the same data region each time
* you request a block. This is very performant, but also forgiving to mistakes. Say you write over the boundaries
* of a block, then you wouldn't notice, since the next time you access the block, the overflow data is (probably)
* still there. Or say an application is relying on flushing the block store in the right moment. Since flushing
* is a no-op in InMemoryBlockStore, you wouldn't notice either.
*
* So this FakeBlockStore has a background copy of each block. When you request a block, you will get a copy of
* the data (instead of a direct pointer as InMemoryBlockStore does) and flushing will copy the data back to the
* background. This way, tests are more likely to fail if they use the blockstore wrongly.
*/
class FakeBlockStore final: public BlockStoreWithRandomKeys {
public:
FakeBlockStore();
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
void updateData(const Key &key, const cpputils::Data &data);
private:
std::map<std::string, cpputils::Data> _blocks;
//This vector keeps a handle of the data regions for all created FakeBlock objects.
//This way, it is ensured that no two created FakeBlock objects will work on the
//same data region. Without this, it could happen that a test case creates a FakeBlock,
//destructs it, creates another one, and the new one gets the same memory region.
//We want to avoid this for the reasons mentioned above (overflow data).
std::vector<std::shared_ptr<cpputils::Data>> _used_dataregions_for_blocks;
mutable std::mutex _mutex;
cpputils::unique_ref<Block> makeFakeBlockFromData(const Key &key, const cpputils::Data &data, bool dirty);
boost::optional<cpputils::unique_ref<Block>> _load(const Key &key);
DISALLOW_COPY_AND_ASSIGN(FakeBlockStore);
};
}
}
#endif

41
src/blockstore/interface/Block.h Normal file
View File

@ -0,0 +1,41 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_INTERFACE_BLOCK_H_
#define MESSMER_BLOCKSTORE_INTERFACE_BLOCK_H_
#include "../utils/Key.h"
#include <cstring>
namespace blockstore {
//TODO Make Block non-virtual class that stores ptr to its blockstore and writes itself back to the blockstore who is offering a corresponding function.
// Then ondisk blockstore can be actually create the file on disk in blockstore::create() and cachingblockstore will delay that call to its base block store.
class Block {
public:
virtual ~Block() {}
virtual const void *data() const = 0;
virtual void write(const void *source, uint64_t offset, uint64_t size) = 0;
virtual void flush() = 0;
virtual size_t size() const = 0;
//TODO Test resize()
virtual void resize(size_t newSize) = 0;
const Key &key() const {
return _key;
}
protected:
Block(const Key &key) : _key(key) {}
private:
const Key _key;
};
}
#endif

39
src/blockstore/interface/BlockStore.h Normal file
View File

@ -0,0 +1,39 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_INTERFACE_BLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_INTERFACE_BLOCKSTORE_H_
#include "Block.h"
#include <string>
#include <boost/optional.hpp>
#include <cpp-utils/pointer/unique_ref.h>
#include <cpp-utils/data/Data.h>
namespace blockstore {
class BlockStore {
public:
virtual ~BlockStore() {}
virtual Key createKey() = 0;
//Returns boost::none if key already exists
virtual boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) = 0;
//TODO Use boost::optional (if key doesn't exist)
// Return nullptr if block with this key doesn't exists
virtual boost::optional<cpputils::unique_ref<Block>> load(const Key &key) = 0;
virtual void remove(cpputils::unique_ref<Block> block) = 0;
virtual uint64_t numBlocks() const = 0;
cpputils::unique_ref<Block> create(const cpputils::Data &data) {
while(true) {
//TODO Copy (data.copy()) necessary?
auto block = tryCreate(createKey(), data.copy());
if (block != boost::none) {
return std::move(*block);
}
}
}
};
}
#endif

1
src/blockstore/interface/helpers/BlockStoreWithRandomKeys.cpp Normal file
View File

@ -0,0 +1 @@
#include "BlockStoreWithRandomKeys.h"

23
src/blockstore/interface/helpers/BlockStoreWithRandomKeys.h Normal file
View File

@ -0,0 +1,23 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_INTERFACE_HELPERS_BLOCKSTOREWITHRANDOMKEYS_H_
#define MESSMER_BLOCKSTORE_INTERFACE_HELPERS_BLOCKSTOREWITHRANDOMKEYS_H_
#include "../BlockStore.h"
#include "../Block.h"
#include <cpp-utils/random/Random.h>
namespace blockstore {
// This is an implementation helpers for BlockStores that use random block keys.
// You should never give this static type to the client. The client should always
// work with the BlockStore interface instead.
class BlockStoreWithRandomKeys: public BlockStore {
public:
Key createKey() final {
return cpputils::Random::PseudoRandom().getFixedSize<Key::BINARY_LENGTH>();
}
};
}
#endif

31
src/blockstore/utils/BlockStoreUtils.cpp Normal file
View File

@ -0,0 +1,31 @@
#include "../interface/BlockStore.h"
#include "BlockStoreUtils.h"
#include <cpp-utils/data/Data.h>
#include <cassert>
#include <cpp-utils/assert/assert.h>
using cpputils::Data;
using cpputils::unique_ref;
namespace blockstore {
namespace utils {
unique_ref<Block> copyToNewBlock(BlockStore *blockStore, const Block &block) {
Data data(block.size());
std::memcpy(data.data(), block.data(), block.size());
return blockStore->create(data);
}
void copyTo(Block *target, const Block &source) {
ASSERT(target->size() == source.size(), "Can't copy block data when blocks have different sizes");
target->write(source.data(), 0, source.size());
}
void fillWithZeroes(Block *target) {
Data zeroes(target->size());
zeroes.FillWithZeroes();
target->write(zeroes.data(), 0, target->size());
}
}
}

19
src/blockstore/utils/BlockStoreUtils.h Normal file
View File

@ -0,0 +1,19 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_UTILS_BLOCKSTOREUTILS_H_
#define MESSMER_BLOCKSTORE_UTILS_BLOCKSTOREUTILS_H_
#include <cpp-utils/pointer/unique_ref.h>
namespace blockstore {
class BlockStore;
class Block;
namespace utils {
cpputils::unique_ref<Block> copyToNewBlock(BlockStore *blockStore, const Block &block);
void copyTo(Block *target, const Block &source);
void fillWithZeroes(Block *target);
}
}
#endif

17
src/blockstore/utils/FileDoesntExistException.cpp Normal file
View File

@ -0,0 +1,17 @@
#include "FileDoesntExistException.h"
namespace bf = boost::filesystem;
using std::runtime_error;
using std::string;
namespace blockstore {
FileDoesntExistException::FileDoesntExistException(const bf::path &filepath)
: runtime_error(string("The file ")+filepath.c_str()+" doesn't exist") {
}
FileDoesntExistException::~FileDoesntExistException() {
}
}

19
src/blockstore/utils/FileDoesntExistException.h Normal file
View File

@ -0,0 +1,19 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_UTILS_FILEDOESNTEXISTEXCEPTION_H_
#define MESSMER_BLOCKSTORE_UTILS_FILEDOESNTEXISTEXCEPTION_H_
#include <boost/filesystem/path.hpp>
#include <stdexcept>
namespace blockstore {
class FileDoesntExistException final: public std::runtime_error {
public:
explicit FileDoesntExistException(const boost::filesystem::path &filepath);
~FileDoesntExistException();
};
}
#endif

1
src/blockstore/utils/Key.cpp Normal file
View File

@ -0,0 +1 @@
#include "Key.h"

32
src/blockstore/utils/Key.h Normal file
View File

@ -0,0 +1,32 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_UTILS_KEY_H_
#define MESSMER_BLOCKSTORE_UTILS_KEY_H_
#include <string>
#include <cpp-utils/data/FixedSizeData.h>
namespace blockstore {
// A key here is NOT a key for encryption, but a key as used in key->value mappings ("access handle for a block").
//TODO Rename to BlockId/BlobId and make it a class containing a FixedSizeData<> member
using Key = cpputils::FixedSizeData<16>;
}
namespace std {
//Allow using blockstore::Key in std::unordered_map / std::unordered_set
template <> struct hash<blockstore::Key> {
size_t operator()(const blockstore::Key &key) const {
//Keys are random, so it is enough to use the first few bytes as a hash
return *(size_t*)(key.data());
}
};
//Allow using blockstore::Key in std::map / std::set
template <> struct less<blockstore::Key> {
bool operator()(const blockstore::Key &lhs, const blockstore::Key &rhs) const {
return 0 > std::memcmp(lhs.data(), rhs.data(), blockstore::Key::BINARY_LENGTH);
}
};
}
#endif

51
src/cpp-utils/CMakeLists.txt Normal file
View File

@ -0,0 +1,51 @@
project (cpp-utils)
set(SOURCES
crypto/symmetric/ciphers.cpp
crypto/kdf/DerivedKey.cpp
crypto/kdf/Scrypt.cpp
crypto/kdf/DerivedKeyConfig.cpp
crypto/RandomPadding.cpp
process/daemonize.cpp
process/subprocess.cpp
tempfile/TempFile.cpp
tempfile/TempDir.cpp
network/HttpClient.cpp
network/CurlHttpClient.cpp
network/FakeHttpClient.cpp
io/Console.cpp
io/pipestream.cpp
thread/LoopThread.cpp
thread/ThreadSystem.cpp
random/Random.cpp
random/RandomGeneratorThread.cpp
random/OSRandomGenerator.cpp
random/PseudoRandomPool.cpp
random/RandomDataBuffer.cpp
random/RandomGenerator.cpp
lock/LockPool.cpp
data/Serializer.cpp
data/Deserializer.cpp
data/DataFixture.cpp
data/DataUtils.cpp
data/Data.cpp
assert/backtrace.cpp
assert/AssertFailed.cpp
)
add_library(${PROJECT_NAME} STATIC ${SOURCES})
# This is needed by boost thread
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PRIVATE rt)
endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PRIVATE pthread curl)
# TODO From Crypto++ 5.7 on, it should support cmake with find_package().
find_library(CryptoPP cryptopp)
target_link_libraries(${PROJECT_NAME} PUBLIC ${CryptoPP} scrypt spdlog)
target_add_boost(${PROJECT_NAME} filesystem system thread)
target_enable_style_warnings(${PROJECT_NAME})
target_activate_cpp14(${PROJECT_NAME})

1
src/cpp-utils/assert/AssertFailed.cpp Normal file
View File

@ -0,0 +1 @@
#include "AssertFailed.h"

25
src/cpp-utils/assert/AssertFailed.h Normal file
View File

@ -0,0 +1,25 @@
#pragma once
#ifndef MESSMER_CPPUTILS_ASSERT_ASSERTFAILED_H
#define MESSMER_CPPUTILS_ASSERT_ASSERTFAILED_H
#include <stdexcept>
#include <string>
#include "../macros.h"
namespace cpputils {
class AssertFailed final: public std::exception {
public:
AssertFailed(const std::string &message) : _message(message) { }
const char *what() const throw() override {
return _message.c_str();
}
private:
std::string _message;
};
}
#endif

45
src/cpp-utils/assert/assert.h Normal file
View File

@ -0,0 +1,45 @@
#pragma once
#ifndef MESSMER_CPPUTILS_ASSERT_ASSERT_H
#define MESSMER_CPPUTILS_ASSERT_ASSERT_H
/**
* This implements an ASSERT(expr, msg) macro.
* In a debug build, it will crash and halt the program on an assert failure.
* In a release build, it will throw an AssertFailed exception instead, which can then be caught.
*/
#include "AssertFailed.h"
#include <iostream>
#include "backtrace.h"
#include "../logging/logging.h"
namespace cpputils {
namespace _assert {
inline std::string format(const char *expr, const std::string &message, const char *file, int line) {
std::string result = std::string()+"Assertion ["+expr+"] failed in "+file+":"+std::to_string(line)+": "+message+"\n\n" + backtrace();
return result;
}
inline void assert_fail_release [[noreturn]] (const char *expr, const std::string &message, const char *file, int line) {
auto msg = format(expr, message, file, line);
using namespace logging;
LOG(ERROR) << msg;
throw AssertFailed(msg);
}
inline void assert_fail_debug [[noreturn]] (const char *expr, const std::string &message, const char *file, int line) {
using namespace logging;
LOG(ERROR) << format(expr, message, file, line);
abort();
}
}
}
#ifdef NDEBUG
//TODO Check whether disabling assertions in prod affects speed.
# define ASSERT(expr, msg) (void)((expr) || (cpputils::_assert::assert_fail_release(#expr, msg, __FILE__, __LINE__),0))
#else
# define ASSERT(expr, msg) (void)((expr) || (cpputils::_assert::assert_fail_debug(#expr, msg, __FILE__, __LINE__),0))
#endif
#endif

68
src/cpp-utils/assert/backtrace.cpp Normal file
View File

@ -0,0 +1,68 @@
#include "backtrace.h"
#include <execinfo.h>
#include <signal.h>
#include <iostream>
#include <unistd.h>
#include <cxxabi.h>
#include <string>
#include <sstream>
#include "../logging/logging.h"
using std::string;
using std::ostringstream;
using namespace cpputils::logging;
//TODO Use the following? https://github.com/bombela/backward-cpp
namespace cpputils {
//TODO Refactor (for example: RAII or at least try{}finally{} instead of free())
std::string demangle(const string &mangledName) {
string result;
int status = -10;
char *demangledName = abi::__cxa_demangle(mangledName.c_str(), NULL, NULL, &status);
if (status == 0) {
result = demangledName;
} else {
result = mangledName;
}
free(demangledName);
return result;
}
std::string pretty(const string &backtraceLine) {
size_t startMangledName = backtraceLine.find('(');
size_t endMangledName = backtraceLine.find('+');
if (startMangledName == string::npos || endMangledName == string::npos) {
return backtraceLine;
}
return demangle(backtraceLine.substr(startMangledName+1, endMangledName-startMangledName-1)) + ": (" + backtraceLine.substr(0, startMangledName) + backtraceLine.substr(endMangledName);
}
string backtrace_to_string(void *array[], size_t size) {
ostringstream result;
char **ptr = backtrace_symbols(array, size);
for (size_t i = 0; i < size; ++i) {
result << pretty(ptr[i]) << "\n";
}
free(ptr);
return result.str();
}
string backtrace() {
constexpr unsigned int MAX_SIZE = 100;
void *array[MAX_SIZE];
size_t size = ::backtrace(array, MAX_SIZE);
return backtrace_to_string(array, size);
}
void sigsegv_handler(int) {
LOG(ERROR) << "SIGSEGV\n" << backtrace();
exit(1);
}
void showBacktraceOnSigSegv() {
signal(SIGSEGV, sigsegv_handler);
}
}

12
src/cpp-utils/assert/backtrace.h Normal file
View File

@ -0,0 +1,12 @@
#pragma once
#ifndef MESSMER_CPPUTILS_ASSERT_BACKTRACE_H
#define MESSMER_CPPUTILS_ASSERT_BACKTRACE_H
#include <string>
namespace cpputils {
std::string backtrace();
void showBacktraceOnSigSegv();
}
#endif

34
src/cpp-utils/crypto/RandomPadding.cpp Normal file
View File

@ -0,0 +1,34 @@
#include "RandomPadding.h"
#include "../logging/logging.h"
#include "../random/Random.h"
using boost::optional;
using namespace cpputils::logging;
namespace cpputils {
Data RandomPadding::add(const Data &data, size_t targetSize) {
uint32_t size = data.size();
if (size >= targetSize - sizeof(size)) {
throw std::runtime_error("Data too large. We should increase padding target size.");
}
Data randomData = Random::PseudoRandom().get(targetSize-sizeof(size)-size);
ASSERT(sizeof(size) + size + randomData.size() == targetSize, "Calculated size of randomData incorrectly");
Data result(targetSize);
std::memcpy(reinterpret_cast<char*>(result.data()), &size, sizeof(size));
std::memcpy(reinterpret_cast<char*>(result.dataOffset(sizeof(size))), reinterpret_cast<const char*>(data.data()), size);
std::memcpy(reinterpret_cast<char*>(result.dataOffset(sizeof(size)+size)), reinterpret_cast<const char*>(randomData.data()), randomData.size());
return result;
}
optional<Data> RandomPadding::remove(const Data &data) {
uint32_t size;
std::memcpy(&size, reinterpret_cast<const char*>(data.data()), sizeof(size));
if(sizeof(size) + size >= data.size()) {
LOG(ERROR) << "Config file is invalid: Invalid padding.";
return boost::none;
};
Data result(size);
std::memcpy(reinterpret_cast<char*>(result.data()), reinterpret_cast<const char*>(data.dataOffset(sizeof(size))), size);
return std::move(result);
}
}

Some files were not shown because too many files have changed in this diff Show More