hardcoresushi
/
libcryfs
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Merge all git repositories into one

This commit is contained in:
Sebastian Messmer 2016-02-11 16:39:42 +01:00
parent 20c617376e 31672e086c
commit c6e8052d93
989 changed files with 204181 additions and 393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
.gitignore vendored
View File

@ -1,35 +1,5 @@
# Compiled Object files
*.slo
*.lo
*.o
*.obj
# Precompiled Headers
*.gch
*.pch
# Compiled Dynamic libraries
*.so
*.dylib
*.dll
# Fortran module files
*.mod
# Compiled Static libraries
*.lai
*.la
*.a
*.lib
# Executables
*.exe
*.out
*.app
# Biicode directory
bii
bin
umltest.inner.sh
umltest.status
/build
/cmake
/.idea

6
.idea/vcs.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

79
.travis.yml
View File

@ -1,17 +1,37 @@
language: cpp
sudo: required
dist: trusty
compiler:
- gcc
- clang
os:
- linux
- osx
matrix:
allow_failures:
- os: osx
- compiler: clang
addons:
apt:
packages:
- libcrypto++-dev
- libfuse-dev
install:
- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/update_gcc_version.sh
&& chmod +x update_gcc_version.sh
&& ./update_gcc_version.sh 4.8
&& rm update_gcc_version.sh
- sudo apt-get install libfuse-dev
# This is needed for packaging 7z distribution packages
- sudo apt-get install software-properties-common && sudo add-apt-repository ppa:george-edison55/precise-backports -y && sudo apt-get update
- sudo apt-get install cmake cmake-data rpm
# CryFS needs cmake >= 3.3, install it.
# Install boost
- wget -O boost.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.56.0/boost_1_56_0.tar.bz2/download
- tar -xf boost.tar.bz2
- cd boost_1_56_0
# TODO We should use clang as toolchain for building boost when clang is used for building our code
- ./bootstrap.sh --with-libraries=filesystem,thread,chrono
- sudo ./b2 -d0 install
- cd ..
- sudo rm -rf boost.tar.bz2 boost_1_56_0
# Install run_with_fuse.sh
- mkdir cmake
- cd cmake
- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/run_with_fuse.sh
- chmod +x run_with_fuse.sh
# Install cmake >= 3.3
- wget --no-check-certificate https://cmake.org/files/v3.3/cmake-3.3.2-Linux-x86_64.tar.gz
&& tar -xf cmake-3.3.2-Linux-x86_64.tar.gz
&& sudo cp -R cmake-3.3.2-Linux-x86_64/* /usr
@ -19,35 +39,14 @@ install:
- cmake --version
# Use /dev/urandom when /dev/random is accessed, because travis doesn't have enough entropy
- sudo cp -a /dev/urandom /dev/random
before_script:
- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/setup_biicode_project.sh
&& chmod +x setup_biicode_project.sh
&& ./setup_biicode_project.sh
&& rm setup_biicode_project.sh
script:
#The configure line is needed as a workaround for the following link, otherwise we wouldn't need "bii configure" at all because "bii build" calls it: http://forum.biicode.com/t/error-could-not-find-the-following-static-boost-libraries-boost-thread/374
- bii cpp:configure || bii cpp:configure
# Build cryfs executable
- bii cpp:build -- -j2
# Build and run test cases
- bii cpp:build --target messmer_cryfs_test_main -- -j2
- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/run_with_fuse.sh
&& chmod +x run_with_fuse.sh
&& ./run_with_fuse.sh "./bin/messmer_cryfs_test_main"
&& rm run_with_fuse.sh
# Make distribution packages
- bii clean
- bii cpp:configure -D CMAKE_BUILD_TYPE=Release
- bii build -- -j2
- cd bii/build/messmer_cryfs && make package && cd ../../..
after_success:
- bii user ${BII_USERNAME} -p ${BII_PASSWORD}
- bii publish
#deploy:
# provider: biicode
# user: ${BII_USERNAME}
# password:
# secure: ${BII_PASSWORD}
# on:
# branch: develop
- cmake ..
- make -j2
- make package -j2
- ./test/cpp-utils/cpp-utils-test
- ./run_with_fuse.sh ./test/fspp/fspp-test
- ./test/parallelaccessstore/parallelaccessstore-test
- ./test/blockstore/blockstore-test
- ./test/blobstore/blobstore-test
after_script:
- rm run_with_fuse.sh

110
CMakeLists.txt
View File

@ -1,26 +1,15 @@
# Earlier cmake versions generate .deb packages for which the package manager says they're bad quality
# and asks the user whether they really want to install it. Cmake 3.3 fixes this.
CMAKE_MINIMUM_REQUIRED(VERSION 3.3)
cmake_minimum_required(VERSION 3.3)
INCLUDE(messmer/cmake/tools)
INCLUDE(messmer/gitversion/cmake)
include(utils.cmake)
SETUP_GOOGLETEST()
require_gcc_version(4.8)
# Actually create targets: EXEcutables and libraries.
ADD_BII_TARGETS()
ACTIVATE_CPP14()
ADD_BOOST(program_options chrono)
ADD_DEFINITIONS(-D_FILE_OFFSET_BITS=64)
GIT_VERSION_INIT()
ENABLE_STYLE_WARNINGS()
SET_TARGET_PROPERTIES(${BII_src_main_TARGET} PROPERTIES OUTPUT_NAME cryfs)
add_subdirectory(vendor)
add_subdirectory(gitversion)
add_subdirectory(src)
add_subdirectory(test)
# Fix debfiles permissions. Unfortunately, git doesn't store file permissions.
# When installing the .deb package and these files have the wrong permissions, the package manager complains.
@ -64,88 +53,3 @@ SET(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/debfiles/pos
INCLUDE(CPack)
# You can safely delete lines from here...
###############################################################################
# REFERENCE #
###############################################################################
#
# This CMakeLists.txt file helps defining your block building and compiling
# To learn more about the CMake use with biicode, visit http://docs.biicode.com/c++.html
#
# ----------------------------------------------------
# NEW FEATURE! Include cmake files from remote blocks:
# -----------------------------------------------------
# Now you can handle cmake dependencies alike you do with c/c++:
#
# INCLUDE(user/block/myrecipe) # include myrecipe.cmake from remote user/block
#
# > EXAMPLE: Include our recipes and activate C++11 in your block (http://www.biicode.com/biicode/cmake)
#
# INCLUDE(biicode/cmake/tools) # Include tools.cmake file from "cmake" block from the "biicode" user
# ACTIVATE_CPP11(INTERFACE ${BII_BLOCK_TARGET})
#
# Remember to run "bii find" to download out cmake tools file
#
# ---------------------
# INIT_BIICODE_BLOCK()
# ---------------------
# This function creates several helper variables as ${BII_BLOCK_NAME} and ${BII_BLOCK_USER}
# Also it loads variables from the cmake/bii_user_block_vars.cmake
# ${BII_LIB_SRC} File list to create the library
# ${BII_LIB_TYPE} Empty (default, STATIC most casess) STATIC or SHARED
# ${BII_LIB_DEPS} Dependencies to other libraries (user2_block2, user3_blockX)
# ${BII_LIB_SYSTEM_HEADERS} System linking requirements as windows.h, pthread.h, etc
#
# You can use or modify them here, for example, to add or remove files from targets based on OS
# Or use typical cmake configurations done BEFORE defining targets. Examples:
# ADD_DEFINITIONS(-DFOO)
# FIND_PACKAGE(OpenGL QUIET)
# You can add INCLUDE_DIRECTORIES here too
#
# ---------------------
# ADD_BIICODE_TARGETS()
# ---------------------
#
# This function creates the following variables:
# ${BII_BLOCK_TARGET} Interface (no files) target for convenient configuration of all
# targets in this block, as the rest of targets always depend on it
# has name in the form "user_block_interface"
# ${BII_LIB_TARGET} Target library name, usually in the form "user_block". May not exist
# if BII_LIB_SRC is empty
# ${BII_BLOCK_TARGETS} List of all targets defined in this block
# ${BII_BLOCK_EXES} List of executables targets defined in this block
# ${BII_exe_name_TARGET}: Executable target (e.g. ${BII_main_TARGET}. You can also use
# directly the name of the executable target (e.g. user_block_main)
#
# > EXAMPLE: Add include directories to all targets of this block
#
# TARGET_INCLUDE_DIRECTORIES(${BII_BLOCK_TARGET} INTERFACE myincludedir)
#
# You can add private include directories to the Lib (if existing)
#
# > EXAMPLE: Link with pthread:
#
# TARGET_LINK_LIBRARIES(${BII_BLOCK_TARGET} INTERFACE pthread)
# or link against library:
# TARGET_LINK_LIBRARIES(${BII_LIB_TARGET} PUBLIC pthread)
# or directly use the library target name:
# TARGET_LINK_LIBRARIES(user_block PUBLIC pthread)
#
# NOTE: This can be also done adding pthread to ${BII_LIB_DEPS}
# BEFORE calling ADD_BIICODE_TARGETS()
#
# > EXAMPLE: how to activate C++11
#
# IF(APPLE)
# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11 -stdlib=libc++")
# ELSEIF (WIN32 OR UNIX)
# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11")
# ENDIF(APPLE)
#
# > EXAMPLE: Set properties to target
#
# SET_TARGET_PROPERTIES(${BII_BLOCK_TARGET} PROPERTIES COMPILE_DEFINITIONS "IOV_MAX=255")
#

165
CMakeLists.txt~ Normal file
View File

@ -0,0 +1,165 @@
<<<<<<< HEAD
# Earlier cmake versions generate .deb packages for which the package manager says they're bad quality
# and asks the user whether they really want to install it. Cmake 3.3 fixes this.
CMAKE_MINIMUM_REQUIRED(VERSION 3.3)
INCLUDE(messmer/cmake/tools)
INCLUDE(messmer/gitversion/cmake)
SETUP_GOOGLETEST()
# Actually create targets: EXEcutables and libraries.
ADD_BII_TARGETS()
ACTIVATE_CPP14()
ADD_BOOST(program_options chrono)
ADD_DEFINITIONS(-D_FILE_OFFSET_BITS=64)
GIT_VERSION_INIT()
ENABLE_STYLE_WARNINGS()
SET_TARGET_PROPERTIES(${BII_src_main_TARGET} PROPERTIES OUTPUT_NAME cryfs)
# Fix debfiles permissions. Unfortunately, git doesn't store file permissions.
# When installing the .deb package and these files have the wrong permissions, the package manager complains.
EXECUTE_PROCESS(COMMAND /bin/bash -c "chmod 0755 ${CMAKE_CURRENT_SOURCE_DIR}/debfiles/*")
INSTALL(TARGETS ${BII_src_main_TARGET}
DESTINATION bin
CONFIGURATIONS Release)
SET(CPACK_GENERATOR TGZ DEB RPM)
SET(CPACK_PACKAGE_NAME "cryfs")
SET(CPACK_PACKAGE_VERSION "${GITVERSION_VERSION_STRING}")
SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Encrypt your files and store them in the cloud.")
SET(CPACK_PACKAGE_DESCRIPTION "CryFS encrypts your files, so you can safely store them anywhere. It works well together with cloud services like Dropbox, iCloud, OneDrive and others.")
SET(CPACK_PACKAGE_CONTACT "Sebastian Messmer <messmer@cryfs.org>")
SET(CPACK_PACKAGE_VENDOR "Sebastian Messmer")
SET(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE")
SET(CPACK_PACKAGE_INSTALL_DIRECTORY "CMake ${CMake_VERSION_MAJOR}.${CMake_VERSION_MINOR}")
IF(WIN32 AND NOT UNIX)
# There is a bug in NSI that does not handle full unix paths properly. Make
# sure there is at least one set of four (4) backlasshes.
#SET(CPACK_PACKAGE_ICON "${CMake_SOURCE_DIR}/Utilities/Release\\\\InstallIcon.bmp")
#SET(CPACK_NSIS_INSTALLED_ICON_NAME "bin\\\\cryfs.exe")
#SET(CPACK_NSIS_DISPLAY_NAME "${CPACK_PACKAGE_INSTALL_DIRECTORY} CryFS")
#SET(CPACK_NSIS_HELP_LINK "http:\\\\\\\\www.cryfs.org")
#SET(CPACK_NSIS_URL_INFO_ABOUT "http:\\\\\\\\www.cryfs.org")
#SET(CPACK_NSIS_CONTACT "messmer@cryfs.org")
#SET(CPACK_NSIS_MODIFY_PATH ON)
ELSE(WIN32 AND NOT UNIX)
SET(CPACK_STRIP_FILES "bin/cryfs")
SET(CPACK_SOURCE_STRIP_FILES "")
ENDIF(WIN32 AND NOT UNIX)
SET(CPACK_PACKAGE_EXECUTABLES "cryfs" "CryFS")
SET(CPACK_DEBIAN_PACKAGE_SECTION "utils")
SET(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
SET(CPACK_DEBIAN_PACKAGE_RECOMMENDS "fuse")
SET(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://www.cryfs.org")
SET(CPACK_RPM_PACKAGE_LICENSE "LGPLv3")
SET(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/debfiles/postinst;${CMAKE_CURRENT_SOURCE_DIR}/debfiles/postrm")
INCLUDE(CPack)
# You can safely delete lines from here...
###############################################################################
# REFERENCE #
###############################################################################
#
# This CMakeLists.txt file helps defining your block building and compiling
# To learn more about the CMake use with biicode, visit http://docs.biicode.com/c++.html
#
# ----------------------------------------------------
# NEW FEATURE! Include cmake files from remote blocks:
# -----------------------------------------------------
# Now you can handle cmake dependencies alike you do with c/c++:
#
# INCLUDE(user/block/myrecipe) # include myrecipe.cmake from remote user/block
#
# > EXAMPLE: Include our recipes and activate C++11 in your block (http://www.biicode.com/biicode/cmake)
#
# INCLUDE(biicode/cmake/tools) # Include tools.cmake file from "cmake" block from the "biicode" user
# ACTIVATE_CPP11(INTERFACE ${BII_BLOCK_TARGET})
#
# Remember to run "bii find" to download out cmake tools file
#
# ---------------------
# INIT_BIICODE_BLOCK()
# ---------------------
# This function creates several helper variables as ${BII_BLOCK_NAME} and ${BII_BLOCK_USER}
# Also it loads variables from the cmake/bii_user_block_vars.cmake
# ${BII_LIB_SRC} File list to create the library
# ${BII_LIB_TYPE} Empty (default, STATIC most casess) STATIC or SHARED
# ${BII_LIB_DEPS} Dependencies to other libraries (user2_block2, user3_blockX)
# ${BII_LIB_SYSTEM_HEADERS} System linking requirements as windows.h, pthread.h, etc
#
# You can use or modify them here, for example, to add or remove files from targets based on OS
# Or use typical cmake configurations done BEFORE defining targets. Examples:
# ADD_DEFINITIONS(-DFOO)
# FIND_PACKAGE(OpenGL QUIET)
# You can add INCLUDE_DIRECTORIES here too
#
# ---------------------
# ADD_BIICODE_TARGETS()
# ---------------------
#
# This function creates the following variables:
# ${BII_BLOCK_TARGET} Interface (no files) target for convenient configuration of all
# targets in this block, as the rest of targets always depend on it
# has name in the form "user_block_interface"
# ${BII_LIB_TARGET} Target library name, usually in the form "user_block". May not exist
# if BII_LIB_SRC is empty
# ${BII_BLOCK_TARGETS} List of all targets defined in this block
# ${BII_BLOCK_EXES} List of executables targets defined in this block
# ${BII_exe_name_TARGET}: Executable target (e.g. ${BII_main_TARGET}. You can also use
# directly the name of the executable target (e.g. user_block_main)
#
# > EXAMPLE: Add include directories to all targets of this block
#
# TARGET_INCLUDE_DIRECTORIES(${BII_BLOCK_TARGET} INTERFACE myincludedir)
#
# You can add private include directories to the Lib (if existing)
#
# > EXAMPLE: Link with pthread:
#
# TARGET_LINK_LIBRARIES(${BII_BLOCK_TARGET} INTERFACE pthread)
# or link against library:
# TARGET_LINK_LIBRARIES(${BII_LIB_TARGET} PUBLIC pthread)
# or directly use the library target name:
# TARGET_LINK_LIBRARIES(user_block PUBLIC pthread)
#
# NOTE: This can be also done adding pthread to ${BII_LIB_DEPS}
# BEFORE calling ADD_BIICODE_TARGETS()
#
# > EXAMPLE: how to activate C++11
#
# IF(APPLE)
# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11 -stdlib=libc++")
# ELSEIF (WIN32 OR UNIX)
# TARGET_COMPILE_OPTIONS(${BII_BLOCK_TARGET} INTERFACE "-std=c++11")
# ENDIF(APPLE)
#
# > EXAMPLE: Set properties to target
#
# SET_TARGET_PROPERTIES(${BII_BLOCK_TARGET} PROPERTIES COMPILE_DEFINITIONS "IOV_MAX=255")
#
=======
# Earlier cmake versions generate .deb packages for which the package manager says they're bad quality
# and asks the user whether they really want to install it. Cmake 3.3 fixes this.
cmake_minimum_required(VERSION 3.3)
include(utils.cmake)
require_gcc_version(4.8)
add_subdirectory(vendor)
add_subdirectory(src)
add_subdirectory(test)
>>>>>>> cpu/cmake

2
README.md
View File

@ -51,7 +51,7 @@ Requirements
- GCC version >= 4.8 or Clang (TODO which minimal version?)
- CMake version >= 3.3
- libcurl4 (including development headers)
- Boost libraries filesystem, system, chrono, thread in version >= 1.56
- Boost libraries filesystem, system, chrono, program_options, thread in version >= 1.56
- Crypto++ >= 5.6.3 (TODO Lower minimal version possible?)
- libFUSE >= 2.8.6 (including development headers)

7
src/CMakeLists.txt Normal file
View File

@ -0,0 +1,7 @@
include_directories(${CMAKE_CURRENT_SOURCE_DIR})
add_subdirectory(cpp-utils)
add_subdirectory(fspp)
add_subdirectory(blockstore)
add_subdirectory(blobstore)
add_subdirectory(cryfs)

29
src/blobstore/CMakeLists.txt Normal file
View File

@ -0,0 +1,29 @@
project (blobstore)
set(SOURCES
implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.cpp
implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.cpp
implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp
implementations/onblocks/utils/Math.cpp
implementations/onblocks/BlobStoreOnBlocks.cpp
implementations/onblocks/datanodestore/DataNode.cpp
implementations/onblocks/datanodestore/DataLeafNode.cpp
implementations/onblocks/datanodestore/DataInnerNode.cpp
implementations/onblocks/datanodestore/DataNodeStore.cpp
implementations/onblocks/datatreestore/impl/algorithms.cpp
implementations/onblocks/datatreestore/DataTree.cpp
implementations/onblocks/datatreestore/DataTreeStore.cpp
implementations/onblocks/BlobOnBlocks.cpp
)
add_library(${PROJECT_NAME} STATIC ${SOURCES})
# This is needed by boost thread
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PRIVATE rt)
endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PUBLIC cpp-utils blockstore)
target_add_boost(${PROJECT_NAME} filesystem system thread)
target_enable_style_warnings(${PROJECT_NAME})
target_activate_cpp14(${PROJECT_NAME})

109
src/blobstore/implementations/onblocks/BlobOnBlocks.cpp Normal file
View File

@ -0,0 +1,109 @@
#include "parallelaccessdatatreestore/DataTreeRef.h"
#include "BlobOnBlocks.h"
#include "datanodestore/DataLeafNode.h"
#include "utils/Math.h"
#include <cmath>
#include <cpp-utils/assert/assert.h>
using std::function;
using cpputils::unique_ref;
using cpputils::Data;
using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataNodeLayout;
using blockstore::Key;
namespace blobstore {
namespace onblocks {
using parallelaccessdatatreestore::DataTreeRef;
BlobOnBlocks::BlobOnBlocks(unique_ref<DataTreeRef> datatree)
: _datatree(std::move(datatree)), _sizeCache(boost::none) {
}
BlobOnBlocks::~BlobOnBlocks() {
}
uint64_t BlobOnBlocks::size() const {
if (_sizeCache == boost::none) {
_sizeCache = _datatree->numStoredBytes();
}
return *_sizeCache;
}
void BlobOnBlocks::resize(uint64_t numBytes) {
_datatree->resizeNumBytes(numBytes);
_sizeCache = numBytes;
}
void BlobOnBlocks::traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function<void (uint64_t, DataLeafNode *leaf, uint32_t, uint32_t)> func) const {
uint64_t endByte = beginByte + sizeBytes;
uint32_t firstLeaf = beginByte / _datatree->maxBytesPerLeaf();
uint32_t endLeaf = utils::ceilDivision(endByte, _datatree->maxBytesPerLeaf());
bool writingOutside = size() < endByte; // TODO Calling size() is slow because it has to traverse the tree
_datatree->traverseLeaves(firstLeaf, endLeaf, [&func, beginByte, endByte, endLeaf, writingOutside](DataLeafNode *leaf, uint32_t leafIndex) {
uint64_t indexOfFirstLeafByte = leafIndex * leaf->maxStoreableBytes();
uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
uint32_t dataEnd = std::min(leaf->maxStoreableBytes(), endByte - indexOfFirstLeafByte);
if (leafIndex == endLeaf-1 && writingOutside) {
// If we are traversing an area that didn't exist before, then the last leaf was just created with a wrong size. We have to fix it.
leaf->resize(dataEnd);
}
func(indexOfFirstLeafByte, leaf, dataBegin, dataEnd-dataBegin);
});
if (writingOutside) {
ASSERT(_datatree->numStoredBytes() == endByte, "Writing didn't grow by the correct number of bytes");
_sizeCache = endByte;
}
}
Data BlobOnBlocks::readAll() const {
//TODO Querying size is inefficient. Is this possible without a call to size()?
uint64_t count = size();
Data result(count);
_read(result.data(), 0, count);
return result;
}
void BlobOnBlocks::read(void *target, uint64_t offset, uint64_t count) const {
ASSERT(offset <= size() && offset + count <= size(), "BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
uint64_t read = tryRead(target, offset, count);
ASSERT(read == count, "BlobOnBlocks::read() couldn't read all requested bytes. Use BlobOnBlocks::tryRead() if this should be allowed.");
}
uint64_t BlobOnBlocks::tryRead(void *target, uint64_t offset, uint64_t count) const {
//TODO Quite inefficient to call size() here, because that has to traverse the tree
uint64_t realCount = std::max(UINT64_C(0), std::min(count, size()-offset));
_read(target, offset, realCount);
return realCount;
}
void BlobOnBlocks::_read(void *target, uint64_t offset, uint64_t count) const {
traverseLeaves(offset, count, [target, offset] (uint64_t indexOfFirstLeafByte, const DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
//TODO Simplify formula, make it easier to understand
leaf->read((uint8_t*)target + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
});
}
void BlobOnBlocks::write(const void *source, uint64_t offset, uint64_t count) {
traverseLeaves(offset, count, [source, offset] (uint64_t indexOfFirstLeafByte, DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
//TODO Simplify formula, make it easier to understand
leaf->write((uint8_t*)source + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
});
}
void BlobOnBlocks::flush() {
_datatree->flush();
}
const Key &BlobOnBlocks::key() const {
return _datatree->key();
}
unique_ref<DataTreeRef> BlobOnBlocks::releaseTree() {
return std::move(_datatree);
}
}
}

52
src/blobstore/implementations/onblocks/BlobOnBlocks.h Normal file
View File

@ -0,0 +1,52 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_BLOBONBLOCKS_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_BLOBONBLOCKS_H_
#include "../../interface/Blob.h"
#include <memory>
#include <boost/optional.hpp>
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataLeafNode;
}
namespace parallelaccessdatatreestore {
class DataTreeRef;
}
class BlobOnBlocks final: public Blob {
public:
BlobOnBlocks(cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> datatree);
~BlobOnBlocks();
const blockstore::Key &key() const override;
uint64_t size() const override;
void resize(uint64_t numBytes) override;
cpputils::Data readAll() const override;
void read(void *target, uint64_t offset, uint64_t size) const override;
uint64_t tryRead(void *target, uint64_t offset, uint64_t size) const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> releaseTree();
private:
void _read(void *target, uint64_t offset, uint64_t count) const;
void traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function<void (uint64_t, datanodestore::DataLeafNode *, uint32_t, uint32_t)>) const;
cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> _datatree;
mutable boost::optional<uint64_t> _sizeCache;
DISALLOW_COPY_AND_ASSIGN(BlobOnBlocks);
};
}
}
#endif

56
src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp Normal file
View File

@ -0,0 +1,56 @@
#include "parallelaccessdatatreestore/DataTreeRef.h"
#include "parallelaccessdatatreestore/ParallelAccessDataTreeStore.h"
#include <blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h>
#include "datanodestore/DataLeafNode.h"
#include "datanodestore/DataNodeStore.h"
#include "datatreestore/DataTreeStore.h"
#include "datatreestore/DataTree.h"
#include "BlobStoreOnBlocks.h"
#include "BlobOnBlocks.h"
#include <cpp-utils/pointer/cast.h>
#include <cpp-utils/assert/assert.h>
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using blockstore::BlockStore;
using blockstore::parallelaccess::ParallelAccessBlockStore;
using blockstore::Key;
using cpputils::dynamic_pointer_move;
using boost::optional;
using boost::none;
namespace blobstore {
namespace onblocks {
using datanodestore::DataNodeStore;
using datatreestore::DataTreeStore;
using parallelaccessdatatreestore::ParallelAccessDataTreeStore;
BlobStoreOnBlocks::BlobStoreOnBlocks(unique_ref<BlockStore> blockStore, uint32_t blocksizeBytes)
: _dataTreeStore(make_unique_ref<ParallelAccessDataTreeStore>(make_unique_ref<DataTreeStore>(make_unique_ref<DataNodeStore>(make_unique_ref<ParallelAccessBlockStore>(std::move(blockStore)), blocksizeBytes)))) {
}
BlobStoreOnBlocks::~BlobStoreOnBlocks() {
}
unique_ref<Blob> BlobStoreOnBlocks::create() {
return make_unique_ref<BlobOnBlocks>(_dataTreeStore->createNewTree());
}
optional<unique_ref<Blob>> BlobStoreOnBlocks::load(const Key &key) {
auto tree = _dataTreeStore->load(key);
if (tree == none) {
return none;
}
return optional<unique_ref<Blob>>(make_unique_ref<BlobOnBlocks>(std::move(*tree)));
}
void BlobStoreOnBlocks::remove(unique_ref<Blob> blob) {
auto _blob = dynamic_pointer_move<BlobOnBlocks>(blob);
ASSERT(_blob != none, "Passed Blob in BlobStoreOnBlocks::remove() is not a BlobOnBlocks.");
_dataTreeStore->remove((*_blob)->releaseTree());
}
}
}

35
src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h Normal file
View File

@ -0,0 +1,35 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_BLOCKED_BLOBSTOREONBLOCKS_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_BLOCKED_BLOBSTOREONBLOCKS_H_
#include "../../interface/BlobStore.h"
#include <blockstore/interface/BlockStore.h>
namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class ParallelAccessDataTreeStore;
}
//TODO Make blobstore able to cope with incomplete data (some blocks missing, because they're not synchronized yet) and write test cases for that
class BlobStoreOnBlocks final: public BlobStore {
public:
BlobStoreOnBlocks(cpputils::unique_ref<blockstore::BlockStore> blockStore, uint32_t blocksizeBytes);
~BlobStoreOnBlocks();
cpputils::unique_ref<Blob> create() override;
boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::Key &key) override;
void remove(cpputils::unique_ref<Blob> blob) override;
private:
cpputils::unique_ref<parallelaccessdatatreestore::ParallelAccessDataTreeStore> _dataTreeStore;
DISALLOW_COPY_AND_ASSIGN(BlobStoreOnBlocks);
};
}
}
#endif

87
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp Normal file
View File

@ -0,0 +1,87 @@
#include "DataInnerNode.h"
#include "DataNodeStore.h"
#include <cpp-utils/assert/assert.h>
using blockstore::Block;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using blockstore::Key;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
DataInnerNode::DataInnerNode(DataNodeView view)
: DataNode(std::move(view)) {
ASSERT(depth() > 0, "Inner node can't have depth 0. Is this a leaf maybe?");
}
DataInnerNode::~DataInnerNode() {
}
unique_ref<DataInnerNode> DataInnerNode::InitializeNewNode(unique_ref<Block> block, const DataNode &first_child) {
DataNodeView node(std::move(block));
node.setDepth(first_child.depth() + 1);
node.setSize(1);
auto result = make_unique_ref<DataInnerNode>(std::move(node));
result->ChildrenBegin()->setKey(first_child.key());
return result;
}
uint32_t DataInnerNode::numChildren() const {
return node().Size();
}
DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->ChildrenBegin());
}
const DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() const {
return node().DataBegin<ChildEntry>();
}
DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->ChildrenEnd());
}
const DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() const {
return ChildrenBegin() + node().Size();
}
DataInnerNode::ChildEntry *DataInnerNode::LastChild() {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->LastChild());
}
const DataInnerNode::ChildEntry *DataInnerNode::LastChild() const {
return getChild(numChildren()-1);
}
DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->getChild(index));
}
const DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) const {
ASSERT(index < numChildren(), "Accessing child out of range");
return ChildrenBegin()+index;
}
void DataInnerNode::addChild(const DataNode &child) {
ASSERT(numChildren() < maxStoreableChildren(), "Adding more children than we can store");
ASSERT(child.depth() == depth()-1, "The child that should be added has wrong depth");
node().setSize(node().Size()+1);
LastChild()->setKey(child.key());
}
void DataInnerNode::removeLastChild() {
ASSERT(node().Size() > 1, "There is no child to remove");
node().setSize(node().Size()-1);
}
uint32_t DataInnerNode::maxStoreableChildren() const {
return node().layout().maxChildrenPerInnerNode();
}
}
}
}

49
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.h Normal file
View File

@ -0,0 +1,49 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_H_
#include "DataNode.h"
#include "DataInnerNode_ChildEntry.h"
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataInnerNode final: public DataNode {
public:
static cpputils::unique_ref<DataInnerNode> InitializeNewNode(cpputils::unique_ref<blockstore::Block> block, const DataNode &first_child_key);
DataInnerNode(DataNodeView block);
~DataInnerNode();
using ChildEntry = DataInnerNode_ChildEntry;
uint32_t maxStoreableChildren() const;
ChildEntry *getChild(unsigned int index);
const ChildEntry *getChild(unsigned int index) const;
uint32_t numChildren() const;
void addChild(const DataNode &child_key);
void removeLastChild();
ChildEntry *LastChild();
const ChildEntry *LastChild() const;
private:
ChildEntry *ChildrenBegin();
ChildEntry *ChildrenEnd();
const ChildEntry *ChildrenBegin() const;
const ChildEntry *ChildrenEnd() const;
DISALLOW_COPY_AND_ASSIGN(DataInnerNode);
};
}
}
}
#endif

30
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode_ChildEntry.h Normal file
View File

@ -0,0 +1,30 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_CHILDENTRY_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATAINNERNODE_CHILDENTRY_H_
#include <cpp-utils/macros.h>
namespace blobstore{
namespace onblocks{
namespace datanodestore{
struct DataInnerNode_ChildEntry final {
public:
blockstore::Key key() const {
return blockstore::Key::FromBinary(_keydata);
}
private:
void setKey(const blockstore::Key &key) {
key.ToBinary(_keydata);
}
friend class DataInnerNode;
uint8_t _keydata[blockstore::Key::BINARY_LENGTH];
DISALLOW_COPY_AND_ASSIGN(DataInnerNode_ChildEntry);
};
}
}
}
#endif

67
src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.cpp Normal file
View File

@ -0,0 +1,67 @@
#include "DataLeafNode.h"
#include "DataInnerNode.h"
#include <cpp-utils/assert/assert.h>
using blockstore::Block;
using cpputils::Data;
using blockstore::Key;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
DataLeafNode::DataLeafNode(DataNodeView view)
: DataNode(std::move(view)) {
ASSERT(node().Depth() == 0, "Leaf node must have depth 0. Is it an inner node instead?");
ASSERT(numBytes() <= maxStoreableBytes(), "Leaf says it stores more bytes than it has space for");
}
DataLeafNode::~DataLeafNode() {
}
unique_ref<DataLeafNode> DataLeafNode::InitializeNewNode(unique_ref<Block> block) {
DataNodeView node(std::move(block));
node.setDepth(0);
node.setSize(0);
//fillDataWithZeroes(); not needed, because a newly created block will be zeroed out. DataLeafNodeTest.SpaceIsZeroFilledWhenGrowing ensures this.
return make_unique_ref<DataLeafNode>(std::move(node));
}
void DataLeafNode::read(void *target, uint64_t offset, uint64_t size) const {
ASSERT(offset <= node().Size() && offset + size <= node().Size(), "Read out of valid area"); // Also check offset, because the addition could lead to overflows
std::memcpy(target, (uint8_t*)node().data() + offset, size);
}
void DataLeafNode::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= node().Size() && offset + size <= node().Size(), "Write out of valid area"); // Also check offset, because the addition could lead to overflows
node().write(source, offset, size);
}
uint32_t DataLeafNode::numBytes() const {
return node().Size();
}
void DataLeafNode::resize(uint32_t new_size) {
ASSERT(new_size <= maxStoreableBytes(), "Trying to resize to a size larger than the maximal size");
uint32_t old_size = node().Size();
if (new_size < old_size) {
fillDataWithZeroesFromTo(new_size, old_size);
}
node().setSize(new_size);
}
void DataLeafNode::fillDataWithZeroesFromTo(off_t begin, off_t end) {
Data ZEROES(end-begin);
ZEROES.FillWithZeroes();
node().write(ZEROES.data(), begin, end-begin);
}
uint64_t DataLeafNode::maxStoreableBytes() const {
return node().layout().maxBytesPerLeaf();
}
}
}
}

39
src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.h Normal file
View File

@ -0,0 +1,39 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATALEAFNODE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATALEAFNODE_H_
#include "DataNode.h"
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataInnerNode;
class DataLeafNode final: public DataNode {
public:
static cpputils::unique_ref<DataLeafNode> InitializeNewNode(cpputils::unique_ref<blockstore::Block> block);
DataLeafNode(DataNodeView block);
~DataLeafNode();
//Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
uint64_t maxStoreableBytes() const;
void read(void *target, uint64_t offset, uint64_t size) const;
void write(const void *source, uint64_t offset, uint64_t size);
uint32_t numBytes() const;
void resize(uint32_t size);
private:
void fillDataWithZeroesFromTo(off_t begin, off_t end);
DISALLOW_COPY_AND_ASSIGN(DataLeafNode);
};
}
}
}
#endif

54
src/blobstore/implementations/onblocks/datanodestore/DataNode.cpp Normal file
View File

@ -0,0 +1,54 @@
#include "DataInnerNode.h"
#include "DataLeafNode.h"
#include "DataNode.h"
#include "DataNodeStore.h"
#include <blockstore/utils/BlockStoreUtils.h>
using blockstore::Block;
using blockstore::Key;
using std::runtime_error;
using cpputils::unique_ref;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
DataNode::DataNode(DataNodeView node)
: _node(std::move(node)) {
}
DataNode::~DataNode() {
}
DataNodeView &DataNode::node() {
return const_cast<DataNodeView&>(const_cast<const DataNode*>(this)->node());
}
const DataNodeView &DataNode::node() const {
return _node;
}
const Key &DataNode::key() const {
return _node.key();
}
uint8_t DataNode::depth() const {
return _node.Depth();
}
unique_ref<DataInnerNode> DataNode::convertToNewInnerNode(unique_ref<DataNode> node, const DataNode &first_child) {
Key key = node->key();
auto block = node->_node.releaseBlock();
blockstore::utils::fillWithZeroes(block.get());
return DataInnerNode::InitializeNewNode(std::move(block), first_child);
}
void DataNode::flush() const {
_node.flush();
}
}
}
}

44
src/blobstore/implementations/onblocks/datanodestore/DataNode.h Normal file
View File

@ -0,0 +1,44 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODE_H_
#include "DataNodeView.h"
#include <cpp-utils/data/Data.h>
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNodeStore;
class DataInnerNode;
class DataNode {
public:
virtual ~DataNode();
const blockstore::Key &key() const;
uint8_t depth() const;
static cpputils::unique_ref<DataInnerNode> convertToNewInnerNode(cpputils::unique_ref<DataNode> node, const DataNode &first_child);
void flush() const;
protected:
DataNode(DataNodeView block);
DataNodeView &node();
const DataNodeView &node() const;
friend class DataNodeStore;
private:
DataNodeView _node;
DISALLOW_COPY_AND_ASSIGN(DataNode);
};
}
}
}
#endif

113
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp Normal file
View File

@ -0,0 +1,113 @@
#include "DataInnerNode.h"
#include "DataLeafNode.h"
#include "DataNodeStore.h"
#include <blockstore/interface/BlockStore.h>
#include <blockstore/interface/Block.h>
#include <blockstore/utils/BlockStoreUtils.h>
#include <cpp-utils/assert/assert.h>
using blockstore::BlockStore;
using blockstore::Block;
using blockstore::Key;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using std::runtime_error;
using boost::optional;
using boost::none;
namespace blobstore {
namespace onblocks {
namespace datanodestore {
DataNodeStore::DataNodeStore(unique_ref<BlockStore> blockstore, uint32_t blocksizeBytes)
: _blockstore(std::move(blockstore)), _layout(blocksizeBytes) {
}
DataNodeStore::~DataNodeStore() {
}
unique_ref<DataNode> DataNodeStore::load(unique_ref<Block> block) {
ASSERT(block->size() == _layout.blocksizeBytes(), "Loading block of wrong size");
DataNodeView node(std::move(block));
if (node.Depth() == 0) {
return make_unique_ref<DataLeafNode>(std::move(node));
} else if (node.Depth() <= MAX_DEPTH) {
return make_unique_ref<DataInnerNode>(std::move(node));
} else {
throw runtime_error("Tree is to deep. Data corruption?");
}
}
unique_ref<DataInnerNode> DataNodeStore::createNewInnerNode(const DataNode &first_child) {
ASSERT(first_child.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
//TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
return DataInnerNode::InitializeNewNode(std::move(block), first_child);
}
unique_ref<DataLeafNode> DataNodeStore::createNewLeafNode() {
//TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
return DataLeafNode::InitializeNewNode(std::move(block));
}
optional<unique_ref<DataNode>> DataNodeStore::load(const Key &key) {
auto block = _blockstore->load(key);
if (block == none) {
return none;
} else {
return load(std::move(*block));
}
}
unique_ref<DataNode> DataNodeStore::createNewNodeAsCopyFrom(const DataNode &source) {
ASSERT(source.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
auto newBlock = blockstore::utils::copyToNewBlock(_blockstore.get(), source.node().block());
return load(std::move(newBlock));
}
unique_ref<DataNode> DataNodeStore::overwriteNodeWith(unique_ref<DataNode> target, const DataNode &source) {
ASSERT(target->node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Target node has wrong layout. Is it from the same DataNodeStore?");
ASSERT(source.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
Key key = target->key();
{
auto targetBlock = target->node().releaseBlock();
cpputils::destruct(std::move(target)); // Call destructor
blockstore::utils::copyTo(targetBlock.get(), source.node().block());
}
auto loaded = load(key);
ASSERT(loaded != none, "Couldn't load the target node after overwriting it");
return std::move(*loaded);
}
void DataNodeStore::remove(unique_ref<DataNode> node) {
auto block = node->node().releaseBlock();
cpputils::destruct(std::move(node)); // Call destructor
_blockstore->remove(std::move(block));
}
uint64_t DataNodeStore::numNodes() const {
return _blockstore->numBlocks();
}
void DataNodeStore::removeSubtree(unique_ref<DataNode> node) {
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(node.get());
if (inner != nullptr) {
for (uint32_t i = 0; i < inner->numChildren(); ++i) {
auto child = load(inner->getChild(i)->key());
ASSERT(child != none, "Couldn't load child node");
removeSubtree(std::move(*child));
}
}
remove(std::move(node));
}
DataNodeLayout DataNodeStore::layout() const {
return _layout;
}
}
}
}

60
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h Normal file
View File

@ -0,0 +1,60 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODESTORE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODESTORE_H_
#include <memory>
#include <cpp-utils/macros.h>
#include "DataNodeView.h"
#include <blockstore/utils/Key.h>
namespace blockstore{
class Block;
class BlockStore;
}
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNode;
class DataLeafNode;
class DataInnerNode;
class DataNodeStore final {
public:
DataNodeStore(cpputils::unique_ref<blockstore::BlockStore> blockstore, uint32_t blocksizeBytes);
~DataNodeStore();
static constexpr uint8_t MAX_DEPTH = 10;
DataNodeLayout layout() const;
boost::optional<cpputils::unique_ref<DataNode>> load(const blockstore::Key &key);
cpputils::unique_ref<DataLeafNode> createNewLeafNode();
cpputils::unique_ref<DataInnerNode> createNewInnerNode(const DataNode &first_child);
cpputils::unique_ref<DataNode> createNewNodeAsCopyFrom(const DataNode &source);
cpputils::unique_ref<DataNode> overwriteNodeWith(cpputils::unique_ref<DataNode> target, const DataNode &source);
void remove(cpputils::unique_ref<DataNode> node);
void removeSubtree(cpputils::unique_ref<DataNode> node);
uint64_t numNodes() const;
//TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree()
private:
cpputils::unique_ref<DataNode> load(cpputils::unique_ref<blockstore::Block> block);
cpputils::unique_ref<blockstore::BlockStore> _blockstore;
const DataNodeLayout _layout;
DISALLOW_COPY_AND_ASSIGN(DataNodeStore);
};
}
}
}
#endif

140
src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h Normal file
View File

@ -0,0 +1,140 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODEVIEW_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATANODESTORE_DATANODEVIEW_H_
#include <blockstore/interface/Block.h>
#include "../BlobStoreOnBlocks.h"
#include "DataInnerNode_ChildEntry.h"
#include <cpp-utils/pointer/unique_ref.h>
#include <memory>
#include <stdexcept>
#include <type_traits>
namespace blobstore {
namespace onblocks {
namespace datanodestore {
//TODO Move DataNodeLayout into own file
class DataNodeLayout final {
public:
constexpr DataNodeLayout(uint32_t blocksizeBytes)
:_blocksizeBytes(
(HEADERSIZE_BYTES + 2*sizeof(DataInnerNode_ChildEntry) <= blocksizeBytes)
? blocksizeBytes
: throw std::logic_error("Blocksize too small, not enough space to store two children in an inner node")) {
}
//Total size of the header
static constexpr uint32_t HEADERSIZE_BYTES = 8;
//Where in the header is the depth field
static constexpr uint32_t DEPTH_OFFSET_BYTES = 0;
//Where in the header is the size field (for inner nodes: number of children, for leafs: content data size)
static constexpr uint32_t SIZE_OFFSET_BYTES = 4;
//Size of a block (header + data region)
constexpr uint32_t blocksizeBytes() const {
return _blocksizeBytes;
}
//Number of bytes in the data region of a node
constexpr uint32_t datasizeBytes() const {
return _blocksizeBytes - HEADERSIZE_BYTES;
}
//Maximum number of children an inner node can store
constexpr uint32_t maxChildrenPerInnerNode() const {
return datasizeBytes() / sizeof(DataInnerNode_ChildEntry);
}
//Maximum number of bytes a leaf can store
//We are returning uint64_t here, because calculations involving maxBytesPerLeaf most probably should use 64bit integers to support blobs >4GB.
constexpr uint64_t maxBytesPerLeaf() const {
return datasizeBytes();
}
private:
uint32_t _blocksizeBytes;
};
class DataNodeView final {
public:
DataNodeView(cpputils::unique_ref<blockstore::Block> block): _block(std::move(block)) {
}
~DataNodeView() {}
DataNodeView(DataNodeView &&rhs) = default;
uint8_t Depth() const {
return *((uint8_t*)_block->data()+DataNodeLayout::DEPTH_OFFSET_BYTES);
}
void setDepth(uint8_t value) {
_block->write(&value, DataNodeLayout::DEPTH_OFFSET_BYTES, sizeof(value));
}
uint32_t Size() const {
return *(uint32_t*)((uint8_t*)_block->data()+DataNodeLayout::SIZE_OFFSET_BYTES);
}
void setSize(uint32_t value) {
_block->write(&value, DataNodeLayout::SIZE_OFFSET_BYTES, sizeof(value));
}
const void *data() const {
return (uint8_t*)_block->data() + DataNodeLayout::HEADERSIZE_BYTES;
}
void write(const void *source, uint64_t offset, uint64_t size) {
_block->write(source, offset + DataNodeLayout::HEADERSIZE_BYTES, size);
}
template<typename Entry>
const Entry *DataBegin() const {
return GetOffset<DataNodeLayout::HEADERSIZE_BYTES, Entry>();
}
template<typename Entry>
const Entry *DataEnd() const {
const unsigned int NUM_ENTRIES = layout().datasizeBytes() / sizeof(Entry);
return DataBegin<Entry>() + NUM_ENTRIES;
}
DataNodeLayout layout() const {
return DataNodeLayout(_block->size());
}
cpputils::unique_ref<blockstore::Block> releaseBlock() {
return std::move(_block);
}
const blockstore::Block &block() const {
return *_block;
}
const blockstore::Key &key() const {
return _block->key();
}
void flush() const {
_block->flush();
}
private:
template<int offset, class Type>
const Type *GetOffset() const {
return (Type*)(((const int8_t*)_block->data())+offset);
}
cpputils::unique_ref<blockstore::Block> _block;
DISALLOW_COPY_AND_ASSIGN(DataNodeView);
};
}
}
}
#endif

334
src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp Normal file
View File

@ -0,0 +1,334 @@
#include "DataTree.h"
#include "../datanodestore/DataNodeStore.h"
#include "../datanodestore/DataInnerNode.h"
#include "../datanodestore/DataLeafNode.h"
#include "../utils/Math.h"
#include "impl/algorithms.h"
#include <cpp-utils/pointer/cast.h>
#include <cpp-utils/pointer/optional_ownership_ptr.h>
#include <cmath>
#include <cpp-utils/assert/assert.h>
using blockstore::Key;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataNodeLayout;
using std::dynamic_pointer_cast;
using std::function;
using boost::shared_mutex;
using boost::shared_lock;
using boost::unique_lock;
using boost::none;
using std::vector;
using cpputils::dynamic_pointer_move;
using cpputils::optional_ownership_ptr;
using cpputils::WithOwnership;
using cpputils::WithoutOwnership;
using cpputils::unique_ref;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
DataTree::DataTree(DataNodeStore *nodeStore, unique_ref<DataNode> rootNode)
: _mutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)) {
}
DataTree::~DataTree() {
}
void DataTree::removeLastDataLeaf() {
auto deletePosOrNull = algorithms::GetLowestRightBorderNodeWithMoreThanOneChildOrNull(_nodeStore, _rootNode.get());
ASSERT(deletePosOrNull.get() != nullptr, "Tree has only one leaf, can't shrink it.");
deleteLastChildSubtree(deletePosOrNull.get());
ifRootHasOnlyOneChildReplaceRootWithItsChild();
}
void DataTree::ifRootHasOnlyOneChildReplaceRootWithItsChild() {
DataInnerNode *rootNode = dynamic_cast<DataInnerNode*>(_rootNode.get());
ASSERT(rootNode != nullptr, "RootNode is not an inner node");
if (rootNode->numChildren() == 1) {
auto child = _nodeStore->load(rootNode->getChild(0)->key());
ASSERT(child != none, "Couldn't load first child of root node");
_rootNode = _nodeStore->overwriteNodeWith(std::move(_rootNode), **child);
_nodeStore->remove(std::move(*child));
}
}
void DataTree::deleteLastChildSubtree(DataInnerNode *node) {
auto lastChild = _nodeStore->load(node->LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
_nodeStore->removeSubtree(std::move(*lastChild));
node->removeLastChild();
}
unique_ref<DataLeafNode> DataTree::addDataLeaf() {
auto insertPosOrNull = algorithms::GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(_nodeStore, _rootNode.get());
if (insertPosOrNull) {
return addDataLeafAt(insertPosOrNull.get());
} else {
return addDataLeafToFullTree();
}
}
unique_ref<DataLeafNode> DataTree::addDataLeafAt(DataInnerNode *insertPos) {
auto new_leaf = _nodeStore->createNewLeafNode();
auto chain = createChainOfInnerNodes(insertPos->depth()-1, new_leaf.get());
insertPos->addChild(*chain);
return new_leaf;
}
optional_ownership_ptr<DataNode> DataTree::createChainOfInnerNodes(unsigned int num, DataNode *child) {
//TODO This function is implemented twice, once with optional_ownership_ptr, once with unique_ref. Redundancy!
optional_ownership_ptr<DataNode> chain = cpputils::WithoutOwnership<DataNode>(child);
for(unsigned int i=0; i<num; ++i) {
auto newnode = _nodeStore->createNewInnerNode(*chain);
chain = cpputils::WithOwnership<DataNode>(std::move(newnode));
}
return chain;
}
unique_ref<DataNode> DataTree::createChainOfInnerNodes(unsigned int num, unique_ref<DataNode> child) {
unique_ref<DataNode> chain = std::move(child);
for(unsigned int i=0; i<num; ++i) {
chain = _nodeStore->createNewInnerNode(*chain);
}
return chain;
}
DataInnerNode* DataTree::increaseTreeDepth(unsigned int levels) {
ASSERT(levels >= 1, "Parameter out of bounds: tried to increase tree depth by zero.");
auto copyOfOldRoot = _nodeStore->createNewNodeAsCopyFrom(*_rootNode);
auto chain = createChainOfInnerNodes(levels-1, copyOfOldRoot.get());
auto newRootNode = DataNode::convertToNewInnerNode(std::move(_rootNode), *chain);
DataInnerNode *result = newRootNode.get();
_rootNode = std::move(newRootNode);
return result;
}
unique_ref<DataLeafNode> DataTree::addDataLeafToFullTree() {
DataInnerNode *rootNode = increaseTreeDepth(1);
auto newLeaf = addDataLeafAt(rootNode);
return newLeaf;
}
const Key &DataTree::key() const {
return _rootNode->key();
}
void DataTree::flush() const {
// By grabbing a lock, we ensure that all modifying functions don't run currently and are therefore flushed
unique_lock<shared_mutex> lock(_mutex);
// We also have to flush the root node
_rootNode->flush();
}
unique_ref<DataNode> DataTree::releaseRootNode() {
return std::move(_rootNode);
}
//TODO Test numLeaves(), for example also two configurations with same number of bytes but different number of leaves (last leaf has 0 bytes)
uint32_t DataTree::numLeaves() const {
shared_lock<shared_mutex> lock(_mutex);
return _numLeaves(*_rootNode);
}
uint32_t DataTree::_numLeaves(const DataNode &node) const {
const DataLeafNode *leaf = dynamic_cast<const DataLeafNode*>(&node);
if (leaf != nullptr) {
return 1;
}
const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(node);
uint64_t numLeavesInLeftChildren = (inner.numChildren()-1) * leavesPerFullChild(inner);
auto lastChild = _nodeStore->load(inner.LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
uint64_t numLeavesInRightChild = _numLeaves(**lastChild);
return numLeavesInLeftChildren + numLeavesInRightChild;
}
void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<void (DataLeafNode*, uint32_t)> func) {
//TODO Can we traverse in parallel?
unique_lock<shared_mutex> lock(_mutex); //TODO Only lock when resizing. Otherwise parallel read/write to a blob is not possible!
ASSERT(beginIndex <= endIndex, "Invalid parameters");
if (0 == endIndex) {
// In this case the utils::ceilLog(_, endIndex) below would fail
return;
}
uint8_t neededTreeDepth = utils::ceilLog(_nodeStore->layout().maxChildrenPerInnerNode(), endIndex);
uint32_t numLeaves = this->_numLeaves(*_rootNode); // TODO Querying the size causes a tree traversal down to the leaves. Possible without querying the size?
if (_rootNode->depth() < neededTreeDepth) {
//TODO Test cases that actually increase it here by 0 level / 1 level / more than 1 level
increaseTreeDepth(neededTreeDepth - _rootNode->depth());
}
if (numLeaves <= beginIndex) {
//TODO Test cases with numLeaves < / >= beginIndex
// There is a gap between the current size and the begin of the traversal
return _traverseLeaves(_rootNode.get(), 0, numLeaves-1, endIndex, [beginIndex, numLeaves, &func, this](DataLeafNode* node, uint32_t index) {
if (index >= beginIndex) {
func(node, index);
} else if (index == numLeaves - 1) {
// It is the old last leaf - resize it to maximum
node->resize(_nodeStore->layout().maxBytesPerLeaf());
}
});
} else if (numLeaves < endIndex) {
// We are starting traversal in the valid region, but traverse until after it (we grow new leaves)
return _traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, [numLeaves, &func, this] (DataLeafNode *node, uint32_t index) {
if (index == numLeaves - 1) {
// It is the old last leaf - resize it to maximum
node->resize(_nodeStore->layout().maxBytesPerLeaf());
}
func(node, index);
});
} else {
//We are traversing entirely inside the valid region
_traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, func);
}
}
void DataTree::_traverseLeaves(DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, function<void (DataLeafNode*, uint32_t)> func) {
DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root);
if (leaf != nullptr) {
ASSERT(beginIndex <= 1 && endIndex <= 1, "If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1.");
if (beginIndex == 0 && endIndex == 1) {
func(leaf, leafOffset);
}
return;
}
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root);
uint32_t leavesPerChild = leavesPerFullChild(*inner);
uint32_t beginChild = beginIndex/leavesPerChild;
uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild);
vector<unique_ref<DataNode>> children = getOrCreateChildren(inner, beginChild, endChild);
for (uint32_t childIndex = beginChild; childIndex < endChild; ++childIndex) {
uint32_t childOffset = childIndex * leavesPerChild;
uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset);
uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
auto child = std::move(children[childIndex-beginChild]);
_traverseLeaves(child.get(), leafOffset + childOffset, localBeginIndex, localEndIndex, func);
}
}
vector<unique_ref<DataNode>> DataTree::getOrCreateChildren(DataInnerNode *node, uint32_t begin, uint32_t end) {
vector<unique_ref<DataNode>> children;
children.reserve(end-begin);
for (uint32_t childIndex = begin; childIndex < std::min(node->numChildren(), end); ++childIndex) {
auto child = _nodeStore->load(node->getChild(childIndex)->key());
ASSERT(child != none, "Couldn't load child node");
children.emplace_back(std::move(*child));
}
for (uint32_t childIndex = node->numChildren(); childIndex < end; ++childIndex) {
//TODO This creates each child with one chain to one leaf only, and then on the next lower level it
// has to create the children for the child. Would be faster to directly create full trees if necessary.
children.emplace_back(addChildTo(node));
}
ASSERT(children.size() == end-begin, "Number of children in the result is wrong");
return children;
}
unique_ref<DataNode> DataTree::addChildTo(DataInnerNode *node) {
auto new_leaf = _nodeStore->createNewLeafNode();
new_leaf->resize(_nodeStore->layout().maxBytesPerLeaf());
auto chain = createChainOfInnerNodes(node->depth()-1, std::move(new_leaf));
node->addChild(*chain);
return std::move(chain);
}
uint32_t DataTree::leavesPerFullChild(const DataInnerNode &root) const {
return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), (uint32_t)root.depth()-1);
}
uint64_t DataTree::numStoredBytes() const {
shared_lock<shared_mutex> lock(_mutex);
return _numStoredBytes();
}
uint64_t DataTree::_numStoredBytes() const {
return _numStoredBytes(*_rootNode);
}
uint64_t DataTree::_numStoredBytes(const DataNode &root) const {
const DataLeafNode *leaf = dynamic_cast<const DataLeafNode*>(&root);
if (leaf != nullptr) {
return leaf->numBytes();
}
const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(root);
uint64_t numBytesInLeftChildren = (inner.numChildren()-1) * leavesPerFullChild(inner) * _nodeStore->layout().maxBytesPerLeaf();
auto lastChild = _nodeStore->load(inner.LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
uint64_t numBytesInRightChild = _numStoredBytes(**lastChild);
return numBytesInLeftChildren + numBytesInRightChild;
}
void DataTree::resizeNumBytes(uint64_t newNumBytes) {
//TODO Can we resize in parallel? Especially creating new blocks (i.e. encrypting them) is expensive and should be done in parallel.
boost::upgrade_lock<shared_mutex> lock(_mutex);
{
boost::upgrade_to_unique_lock<shared_mutex> exclusiveLock(lock);
//TODO Faster implementation possible (no addDataLeaf()/removeLastDataLeaf() in a loop, but directly resizing)
LastLeaf(_rootNode.get())->resize(_nodeStore->layout().maxBytesPerLeaf());
uint64_t currentNumBytes = _numStoredBytes();
ASSERT(currentNumBytes % _nodeStore->layout().maxBytesPerLeaf() == 0, "The last leaf is not a max data leaf, although we just resized it to be one.");
uint32_t currentNumLeaves = currentNumBytes / _nodeStore->layout().maxBytesPerLeaf();
uint32_t newNumLeaves = std::max(UINT64_C(1), utils::ceilDivision(newNumBytes, _nodeStore->layout().maxBytesPerLeaf()));
for(uint32_t i = currentNumLeaves; i < newNumLeaves; ++i) {
addDataLeaf()->resize(_nodeStore->layout().maxBytesPerLeaf());
}
for(uint32_t i = currentNumLeaves; i > newNumLeaves; --i) {
removeLastDataLeaf();
}
uint32_t newLastLeafSize = newNumBytes - (newNumLeaves-1)*_nodeStore->layout().maxBytesPerLeaf();
LastLeaf(_rootNode.get())->resize(newLastLeafSize);
}
ASSERT(newNumBytes == _numStoredBytes(), "We resized to the wrong number of bytes ("+std::to_string(numStoredBytes())+" instead of "+std::to_string(newNumBytes)+")");
}
optional_ownership_ptr<DataLeafNode> DataTree::LastLeaf(DataNode *root) {
DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root);
if (leaf != nullptr) {
return WithoutOwnership(leaf);
}
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root);
auto lastChild = _nodeStore->load(inner->LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
return WithOwnership(LastLeaf(std::move(*lastChild)));
}
unique_ref<DataLeafNode> DataTree::LastLeaf(unique_ref<DataNode> root) {
auto leaf = dynamic_pointer_move<DataLeafNode>(root);
if (leaf != none) {
return std::move(*leaf);
}
auto inner = dynamic_pointer_move<DataInnerNode>(root);
ASSERT(inner != none, "Root node is neither a leaf nor an inner node");
auto child = _nodeStore->load((*inner)->LastChild()->key());
ASSERT(child != none, "Couldn't load last child");
return LastLeaf(std::move(*child));
}
uint64_t DataTree::maxBytesPerLeaf() const {
return _nodeStore->layout().maxBytesPerLeaf();
}
}
}
}

79
src/blobstore/implementations/onblocks/datatreestore/DataTree.h Normal file
View File

@ -0,0 +1,79 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREE_H_
#include <memory>
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/optional_ownership_ptr.h>
#include "../datanodestore/DataNodeView.h"
//TODO Replace with C++14 once std::shared_mutex is supported
#include <boost/thread/shared_mutex.hpp>
#include <blockstore/utils/Key.h>
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNodeStore;
class DataInnerNode;
class DataLeafNode;
class DataNode;
}
namespace datatreestore {
//TODO It is strange that DataLeafNode is still part in the public interface of DataTree. This should be separated somehow.
class DataTree final {
public:
DataTree(datanodestore::DataNodeStore *nodeStore, cpputils::unique_ref<datanodestore::DataNode> rootNode);
~DataTree();
const blockstore::Key &key() const;
//Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
uint64_t maxBytesPerLeaf() const;
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func);
void resizeNumBytes(uint64_t newNumBytes);
uint32_t numLeaves() const;
uint64_t numStoredBytes() const;
void flush() const;
private:
mutable boost::shared_mutex _mutex;
datanodestore::DataNodeStore *_nodeStore;
cpputils::unique_ref<datanodestore::DataNode> _rootNode;
cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeaf();
void removeLastDataLeaf();
cpputils::unique_ref<datanodestore::DataNode> releaseRootNode();
friend class DataTreeStore;
cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeafAt(datanodestore::DataInnerNode *insertPos);
cpputils::optional_ownership_ptr<datanodestore::DataNode> createChainOfInnerNodes(unsigned int num, datanodestore::DataNode *child);
cpputils::unique_ref<datanodestore::DataNode> createChainOfInnerNodes(unsigned int num, cpputils::unique_ref<datanodestore::DataNode> child);
cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeafToFullTree();
void deleteLastChildSubtree(datanodestore::DataInnerNode *node);
void ifRootHasOnlyOneChildReplaceRootWithItsChild();
//TODO Use underscore for private methods
void _traverseLeaves(datanodestore::DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func);
uint32_t leavesPerFullChild(const datanodestore::DataInnerNode &root) const;
uint64_t _numStoredBytes() const;
uint64_t _numStoredBytes(const datanodestore::DataNode &root) const;
uint32_t _numLeaves(const datanodestore::DataNode &node) const;
cpputils::optional_ownership_ptr<datanodestore::DataLeafNode> LastLeaf(datanodestore::DataNode *root);
cpputils::unique_ref<datanodestore::DataLeafNode> LastLeaf(cpputils::unique_ref<datanodestore::DataNode> root);
datanodestore::DataInnerNode* increaseTreeDepth(unsigned int levels);
std::vector<cpputils::unique_ref<datanodestore::DataNode>> getOrCreateChildren(datanodestore::DataInnerNode *node, uint32_t begin, uint32_t end);
cpputils::unique_ref<datanodestore::DataNode> addChildTo(datanodestore::DataInnerNode *node);
DISALLOW_COPY_AND_ASSIGN(DataTree);
};
}
}
}
#endif

46
src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.cpp Normal file
View File

@ -0,0 +1,46 @@
#include "DataTreeStore.h"
#include "../datanodestore/DataNodeStore.h"
#include "../datanodestore/DataLeafNode.h"
#include "DataTree.h"
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blobstore::onblocks::datanodestore::DataNode;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
DataTreeStore::DataTreeStore(unique_ref<DataNodeStore> nodeStore)
: _nodeStore(std::move(nodeStore)) {
}
DataTreeStore::~DataTreeStore() {
}
optional<unique_ref<DataTree>> DataTreeStore::load(const blockstore::Key &key) {
auto node = _nodeStore->load(key);
if (node == none) {
return none;
}
return make_unique_ref<DataTree>(_nodeStore.get(), std::move(*node));
}
unique_ref<DataTree> DataTreeStore::createNewTree() {
auto newleaf = _nodeStore->createNewLeafNode();
return make_unique_ref<DataTree>(_nodeStore.get(), std::move(newleaf));
}
void DataTreeStore::remove(unique_ref<DataTree> tree) {
auto root = tree->releaseRootNode();
cpputils::destruct(std::move(tree)); // Destruct tree
_nodeStore->removeSubtree(std::move(root));
}
}
}
}

40
src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h Normal file
View File

@ -0,0 +1,40 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_DATATREESTORE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_DATATREESTORE_H_
#include <memory>
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/unique_ref.h>
#include <blockstore/utils/Key.h>
#include <boost/optional.hpp>
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNodeStore;
}
namespace datatreestore {
class DataTree;
class DataTreeStore final {
public:
DataTreeStore(cpputils::unique_ref<datanodestore::DataNodeStore> nodeStore);
~DataTreeStore();
boost::optional<cpputils::unique_ref<DataTree>> load(const blockstore::Key &key);
cpputils::unique_ref<DataTree> createNewTree();
void remove(cpputils::unique_ref<DataTree> tree);
private:
cpputils::unique_ref<datanodestore::DataNodeStore> _nodeStore;
DISALLOW_COPY_AND_ASSIGN(DataTreeStore);
};
}
}
}
#endif

67
src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.cpp Normal file
View File

@ -0,0 +1,67 @@
#include "algorithms.h"
#include <cpp-utils/pointer/cast.h>
#include <blockstore/utils/Key.h>
#include "../../datanodestore/DataInnerNode.h"
#include "../../datanodestore/DataNodeStore.h"
#include <cpp-utils/assert/assert.h>
using std::function;
using cpputils::optional_ownership_ptr;
using cpputils::dynamic_pointer_move;
using cpputils::unique_ref;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blockstore::Key;
using boost::optional;
using boost::none;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
namespace algorithms {
optional<unique_ref<DataInnerNode>> getLastChildAsInnerNode(DataNodeStore *nodeStore, const DataInnerNode &node) {
Key key = node.LastChild()->key();
auto lastChild = nodeStore->load(key);
ASSERT(lastChild != none, "Couldn't load last child");
return dynamic_pointer_move<DataInnerNode>(*lastChild);
}
//Returns the lowest right border node meeting the condition specified (exclusive the leaf).
//Returns nullptr, if no inner right border node meets the condition.
optional_ownership_ptr<DataInnerNode> GetLowestInnerRightBorderNodeWithConditionOrNull(DataNodeStore *nodeStore, datanodestore::DataNode *rootNode, function<bool(const DataInnerNode &)> condition) {
optional_ownership_ptr<DataInnerNode> currentNode = cpputils::WithoutOwnership(dynamic_cast<DataInnerNode*>(rootNode));
optional_ownership_ptr<DataInnerNode> result = cpputils::null<DataInnerNode>();
for (unsigned int i=0; i < rootNode->depth(); ++i) {
//TODO This unnecessarily loads the leaf node in the last loop run
auto lastChild = getLastChildAsInnerNode(nodeStore, *currentNode);
if (condition(*currentNode)) {
result = std::move(currentNode);
}
ASSERT(lastChild != none || static_cast<int>(i) == rootNode->depth()-1, "Couldn't get last child as inner node but we're not deep enough yet for the last child to be a leaf");
if (lastChild != none) {
currentNode = cpputils::WithOwnership(std::move(*lastChild));
}
}
return result;
}
optional_ownership_ptr<DataInnerNode> GetLowestRightBorderNodeWithMoreThanOneChildOrNull(DataNodeStore *nodeStore, DataNode *rootNode) {
return GetLowestInnerRightBorderNodeWithConditionOrNull(nodeStore, rootNode, [] (const datanodestore::DataInnerNode &node) {
return node.numChildren() > 1;
});
}
optional_ownership_ptr<datanodestore::DataInnerNode> GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode) {
return GetLowestInnerRightBorderNodeWithConditionOrNull(nodeStore, rootNode, [] (const datanodestore::DataInnerNode &node) {
return node.numChildren() < node.maxStoreableChildren();
});
}
}
}
}
}

30
src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.h Normal file
View File

@ -0,0 +1,30 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_IMPL_ALGORITHMS_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_DATATREESTORE_IMPL_ALGORITHMS_H_
#include <cpp-utils/pointer/optional_ownership_ptr.h>
namespace blobstore {
namespace onblocks {
namespace datanodestore{
class DataNode;
class DataInnerNode;
class DataNodeStore;
}
namespace datatreestore {
namespace algorithms {
//Returns the lowest right border node with at least two children.
//Returns nullptr, if all right border nodes have only one child (since the root is a right border node, this means that the whole tree has exactly one leaf)
cpputils::optional_ownership_ptr<datanodestore::DataInnerNode> GetLowestRightBorderNodeWithMoreThanOneChildOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode);
//Returns the lowest right border node with less than k children (not considering leaves).
//Returns nullptr, if all right border nodes have k children (the tree is full)
cpputils::optional_ownership_ptr<datanodestore::DataInnerNode> GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(datanodestore::DataNodeStore *nodeStore, datanodestore::DataNode *rootNode);
}
}
}
}
#endif

1
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.cpp Normal file
View File

@ -0,0 +1 @@
#include "DataTreeRef.h"

55
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h Normal file
View File

@ -0,0 +1,55 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_DATATREEREF_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_DATATREEREF_H_
#include <parallelaccessstore/ParallelAccessStore.h>
#include "../datatreestore/DataTree.h"
namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class DataTreeRef final: public parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::Key>::ResourceRefBase {
public:
DataTreeRef(datatreestore::DataTree *baseTree): _baseTree(baseTree) {}
const blockstore::Key &key() const {
return _baseTree->key();
}
uint64_t maxBytesPerLeaf() const {
return _baseTree->maxBytesPerLeaf();
}
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func) {
return _baseTree->traverseLeaves(beginIndex, endIndex, func);
}
uint32_t numLeaves() const {
return _baseTree->numLeaves();
}
void resizeNumBytes(uint64_t newNumBytes) {
return _baseTree->resizeNumBytes(newNumBytes);
}
uint64_t numStoredBytes() const {
return _baseTree->numStoredBytes();
}
void flush() {
return _baseTree->flush();
}
private:
datatreestore::DataTree *_baseTree;
DISALLOW_COPY_AND_ASSIGN(DataTreeRef);
};
}
}
}
#endif

44
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp Normal file
View File

@ -0,0 +1,44 @@
#include "DataTreeRef.h"
#include "ParallelAccessDataTreeStore.h"
#include "ParallelAccessDataTreeStoreAdapter.h"
#include "../datanodestore/DataNodeStore.h"
#include "../datanodestore/DataLeafNode.h"
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using blobstore::onblocks::datatreestore::DataTreeStore;
using blockstore::Key;
namespace blobstore {
namespace onblocks {
using datatreestore::DataTreeStore;
using datatreestore::DataTree;
namespace parallelaccessdatatreestore {
ParallelAccessDataTreeStore::ParallelAccessDataTreeStore(unique_ref<DataTreeStore> dataTreeStore)
: _dataTreeStore(std::move(dataTreeStore)), _parallelAccessStore(make_unique_ref<ParallelAccessDataTreeStoreAdapter>(_dataTreeStore.get())) {
}
ParallelAccessDataTreeStore::~ParallelAccessDataTreeStore() {
}
optional<unique_ref<DataTreeRef>> ParallelAccessDataTreeStore::load(const blockstore::Key &key) {
return _parallelAccessStore.load(key);
}
unique_ref<DataTreeRef> ParallelAccessDataTreeStore::createNewTree() {
auto dataTree = _dataTreeStore->createNewTree();
Key key = dataTree->key();
return _parallelAccessStore.add(key, std::move(dataTree));
}
void ParallelAccessDataTreeStore::remove(unique_ref<DataTreeRef> tree) {
Key key = tree->key();
return _parallelAccessStore.remove(key, std::move(tree));
}
}
}
}

43
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h Normal file
View File

@ -0,0 +1,43 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTORE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTORE_H_
#include <memory>
#include <cpp-utils/macros.h>
#include <blockstore/utils/Key.h>
#include <parallelaccessstore/ParallelAccessStore.h>
namespace blobstore {
namespace onblocks {
namespace datatreestore {
class DataTreeStore;
class DataTree;
}
namespace parallelaccessdatatreestore {
class DataTreeRef;
//TODO Test CachingDataTreeStore
class ParallelAccessDataTreeStore final {
public:
ParallelAccessDataTreeStore(cpputils::unique_ref<datatreestore::DataTreeStore> dataTreeStore);
~ParallelAccessDataTreeStore();
boost::optional<cpputils::unique_ref<DataTreeRef>> load(const blockstore::Key &key);
cpputils::unique_ref<DataTreeRef> createNewTree();
void remove(cpputils::unique_ref<DataTreeRef> tree);
private:
cpputils::unique_ref<datatreestore::DataTreeStore> _dataTreeStore;
parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::Key> _parallelAccessStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessDataTreeStore);
};
}
}
}
#endif

1
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.cpp Normal file
View File

@ -0,0 +1 @@
#include "ParallelAccessDataTreeStoreAdapter.h"

38
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.h Normal file
View File

@ -0,0 +1,38 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTOREADAPTER_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_PARALLELACCESSDATATREESTORE_PARALLELACCESSDATATREESTOREADAPTER_H_
#include <cpp-utils/macros.h>
#include <parallelaccessstore/ParallelAccessStore.h>
#include "../datatreestore/DataTreeStore.h"
#include "../datatreestore/DataTree.h"
namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class ParallelAccessDataTreeStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore<datatreestore::DataTree, blockstore::Key> {
public:
ParallelAccessDataTreeStoreAdapter(datatreestore::DataTreeStore *baseDataTreeStore)
:_baseDataTreeStore(std::move(baseDataTreeStore)) {
}
boost::optional<cpputils::unique_ref<datatreestore::DataTree>> loadFromBaseStore(const blockstore::Key &key) override {
return _baseDataTreeStore->load(key);
}
void removeFromBaseStore(cpputils::unique_ref<datatreestore::DataTree> dataTree) override {
return _baseDataTreeStore->remove(std::move(dataTree));
}
private:
datatreestore::DataTreeStore *_baseDataTreeStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessDataTreeStoreAdapter);
};
}
}
}
#endif

1
src/blobstore/implementations/onblocks/utils/Math.cpp Normal file
View File

@ -0,0 +1 @@
#include "Math.h"

43
src/blobstore/implementations/onblocks/utils/Math.h Normal file
View File

@ -0,0 +1,43 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_UTILS_MATH_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_UTILS_MATH_H_
#include <cstdint>
#include <cmath>
namespace blobstore {
namespace onblocks {
namespace utils {
template<typename INT_TYPE>
inline INT_TYPE intPow(INT_TYPE base, INT_TYPE exponent) {
INT_TYPE result = 1;
for(INT_TYPE i = 0; i < exponent; ++i) {
result *= base;
}
return result;
}
template<typename INT_TYPE>
inline INT_TYPE ceilDivision(INT_TYPE dividend, INT_TYPE divisor) {
return (dividend + divisor - 1)/divisor;
}
template<typename INT_TYPE>
inline INT_TYPE maxZeroSubtraction(INT_TYPE minuend, INT_TYPE subtrahend) {
if (minuend < subtrahend) {
return 0u;
}
return minuend-subtrahend;
}
template<typename INT_TYPE>
inline INT_TYPE ceilLog(INT_TYPE base, INT_TYPE value) {
return std::ceil((long double)std::log(value)/(long double)std::log(base));
}
}
}
}
#endif

35
src/blobstore/interface/Blob.h Normal file
View File

@ -0,0 +1,35 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_INTERFACE_BLOB_H_
#define MESSMER_BLOBSTORE_INTERFACE_BLOB_H_
#include <cstring>
#include <cstdint>
#include <blockstore/utils/Key.h>
#include <cpp-utils/data/Data.h>
namespace blobstore {
class Blob {
public:
virtual ~Blob() {}
//TODO Use own Key class for blobstore
virtual const blockstore::Key &key() const = 0;
virtual uint64_t size() const = 0;
virtual void resize(uint64_t numBytes) = 0;
virtual cpputils::Data readAll() const = 0;
virtual void read(void *target, uint64_t offset, uint64_t size) const = 0;
virtual uint64_t tryRead(void *target, uint64_t offset, uint64_t size) const = 0;
virtual void write(const void *source, uint64_t offset, uint64_t size) = 0;
virtual void flush() = 0;
//TODO Test tryRead
};
}
#endif

25
src/blobstore/interface/BlobStore.h Normal file
View File

@ -0,0 +1,25 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_INTERFACE_BLOBSTORE_H_
#define MESSMER_BLOBSTORE_INTERFACE_BLOBSTORE_H_
#include "Blob.h"
#include <string>
#include <memory>
#include <blockstore/utils/Key.h>
#include <cpp-utils/pointer/unique_ref.h>
namespace blobstore {
class BlobStore {
public:
virtual ~BlobStore() {}
virtual cpputils::unique_ref<Blob> create() = 0;
virtual boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::Key &key) = 0;
virtual void remove(cpputils::unique_ref<Blob> blob) = 0;
};
}
#endif

43
src/blockstore/CMakeLists.txt Normal file
View File

@ -0,0 +1,43 @@
project (blockstore)
set(SOURCES
utils/Key.cpp
utils/BlockStoreUtils.cpp
utils/FileDoesntExistException.cpp
interface/helpers/BlockStoreWithRandomKeys.cpp
implementations/testfake/FakeBlockStore.cpp
implementations/testfake/FakeBlock.cpp
implementations/inmemory/InMemoryBlock.cpp
implementations/inmemory/InMemoryBlockStore.cpp
implementations/parallelaccess/ParallelAccessBlockStore.cpp
implementations/parallelaccess/BlockRef.cpp
implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp
implementations/compressing/CompressingBlockStore.cpp
implementations/compressing/CompressedBlock.cpp
implementations/compressing/compressors/RunLengthEncoding.cpp
implementations/compressing/compressors/Gzip.cpp
implementations/encrypted/EncryptedBlockStore.cpp
implementations/encrypted/EncryptedBlock.cpp
implementations/ondisk/OnDiskBlockStore.cpp
implementations/ondisk/OnDiskBlock.cpp
implementations/caching/CachingBlockStore.cpp
implementations/caching/cache/PeriodicTask.cpp
implementations/caching/cache/CacheEntry.cpp
implementations/caching/cache/Cache.cpp
implementations/caching/cache/QueueMap.cpp
implementations/caching/CachedBlock.cpp
implementations/caching/NewBlock.cpp
)
add_library(${PROJECT_NAME} STATIC ${SOURCES})
# This is needed by boost thread
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PRIVATE rt)
endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PUBLIC cpp-utils)
target_add_boost(${PROJECT_NAME} filesystem system thread)
target_enable_style_warnings(${PROJECT_NAME})
target_activate_cpp14(${PROJECT_NAME})

46
src/blockstore/implementations/caching/CachedBlock.cpp Normal file
View File

@ -0,0 +1,46 @@
#include "CachedBlock.h"
#include "CachingBlockStore.h"
using cpputils::unique_ref;
namespace blockstore {
namespace caching {
CachedBlock::CachedBlock(unique_ref<Block> baseBlock, CachingBlockStore *blockStore)
:Block(baseBlock->key()),
_blockStore(blockStore),
_baseBlock(std::move(baseBlock)) {
}
CachedBlock::~CachedBlock() {
if (_baseBlock.get() != nullptr) {
_blockStore->release(std::move(_baseBlock));
}
}
const void *CachedBlock::data() const {
return _baseBlock->data();
}
void CachedBlock::write(const void *source, uint64_t offset, uint64_t size) {
return _baseBlock->write(source, offset, size);
}
void CachedBlock::flush() {
return _baseBlock->flush();
}
size_t CachedBlock::size() const {
return _baseBlock->size();
}
void CachedBlock::resize(size_t newSize) {
return _baseBlock->resize(newSize);
}
unique_ref<Block> CachedBlock::releaseBlock() {
return std::move(_baseBlock);
}
}
}

39
src/blockstore/implementations/caching/CachedBlock.h Normal file
View File

@ -0,0 +1,39 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/pointer/unique_ref.h>
namespace blockstore {
namespace caching {
class CachingBlockStore;
class CachedBlock final: public Block {
public:
//TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
CachedBlock(cpputils::unique_ref<Block> baseBlock, CachingBlockStore *blockStore);
~CachedBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref<Block> releaseBlock();
private:
CachingBlockStore *_blockStore;
cpputils::unique_ref<Block> _baseBlock;
DISALLOW_COPY_AND_ASSIGN(CachedBlock);
};
}
}
#endif

92
src/blockstore/implementations/caching/CachingBlockStore.cpp Normal file
View File

@ -0,0 +1,92 @@
#include "CachedBlock.h"
#include "NewBlock.h"
#include "CachingBlockStore.h"
#include "../../interface/Block.h"
#include <algorithm>
#include <cpp-utils/pointer/cast.h>
#include <cpp-utils/assert/assert.h>
using cpputils::dynamic_pointer_move;
using cpputils::Data;
using boost::optional;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::none;
namespace blockstore {
namespace caching {
CachingBlockStore::CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore)
:_baseBlockStore(std::move(baseBlockStore)), _cache(), _numNewBlocks(0) {
}
Key CachingBlockStore::createKey() {
return _baseBlockStore->createKey();
}
optional<unique_ref<Block>> CachingBlockStore::tryCreate(const Key &key, Data data) {
ASSERT(_cache.pop(key) == none, "Key already exists in cache");
//TODO Shouldn't we return boost::none if the key already exists?
//TODO Key can also already exist but not be in the cache right now.
++_numNewBlocks;
return unique_ref<Block>(make_unique_ref<CachedBlock>(make_unique_ref<NewBlock>(key, std::move(data), this), this));
}
optional<unique_ref<Block>> CachingBlockStore::load(const Key &key) {
optional<unique_ref<Block>> optBlock = _cache.pop(key);
//TODO an optional<> class with .getOrElse() would make this code simpler. boost::optional<>::value_or_eval didn't seem to work with unique_ptr members.
if (optBlock != none) {
return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*optBlock), this));
} else {
auto block = _baseBlockStore->load(key);
if (block == none) {
return none;
} else {
return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*block), this));
}
}
}
void CachingBlockStore::remove(cpputils::unique_ref<Block> block) {
auto cached_block = dynamic_pointer_move<CachedBlock>(block);
ASSERT(cached_block != none, "Passed block is not a CachedBlock");
auto baseBlock = (*cached_block)->releaseBlock();
auto baseNewBlock = dynamic_pointer_move<NewBlock>(baseBlock);
if (baseNewBlock != none) {
if(!(*baseNewBlock)->alreadyExistsInBaseStore()) {
--_numNewBlocks;
}
(*baseNewBlock)->remove();
} else {
_baseBlockStore->remove(std::move(baseBlock));
}
}
uint64_t CachingBlockStore::numBlocks() const {
return _baseBlockStore->numBlocks() + _numNewBlocks;
}
void CachingBlockStore::release(unique_ref<Block> block) {
Key key = block->key();
_cache.push(key, std::move(block));
}
optional<unique_ref<Block>> CachingBlockStore::tryCreateInBaseStore(const Key &key, Data data) {
auto block = _baseBlockStore->tryCreate(key, std::move(data));
if (block != none) {
--_numNewBlocks;
}
return block;
}
void CachingBlockStore::removeFromBaseStore(cpputils::unique_ref<Block> block) {
_baseBlockStore->remove(std::move(block));
}
void CachingBlockStore::flush() {
_cache.flush();
}
}
}

40
src/blockstore/implementations/caching/CachingBlockStore.h Normal file
View File

@ -0,0 +1,40 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
#include "cache/Cache.h"
#include "../../interface/BlockStore.h"
namespace blockstore {
namespace caching {
//TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel)
class CachingBlockStore final: public BlockStore {
public:
explicit CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
void release(cpputils::unique_ref<Block> block);
boost::optional<cpputils::unique_ref<Block>> tryCreateInBaseStore(const Key &key, cpputils::Data data);
void removeFromBaseStore(cpputils::unique_ref<Block> block);
void flush();
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
Cache<Key, cpputils::unique_ref<Block>, 1000> _cache;
uint32_t _numNewBlocks;
DISALLOW_COPY_AND_ASSIGN(CachingBlockStore);
};
}
}
#endif

75
src/blockstore/implementations/caching/NewBlock.cpp Normal file
View File

@ -0,0 +1,75 @@
#include "NewBlock.h"
#include "CachingBlockStore.h"
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
using cpputils::Data;
using boost::none;
namespace blockstore {
namespace caching {
NewBlock::NewBlock(const Key &key, Data data, CachingBlockStore *blockStore)
:Block(key),
_blockStore(blockStore),
_data(std::move(data)),
_baseBlock(none),
_dataChanged(true) {
}
NewBlock::~NewBlock() {
writeToBaseBlockIfChanged();
}
const void *NewBlock::data() const {
return _data.data();
}
void NewBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area");
std::memcpy((uint8_t*)_data.data()+offset, source, size);
_dataChanged = true;
}
void NewBlock::writeToBaseBlockIfChanged() {
if (_dataChanged) {
if (_baseBlock == none) {
//TODO _data.copy() necessary?
auto newBase = _blockStore->tryCreateInBaseStore(key(), _data.copy());
ASSERT(newBase != boost::none, "Couldn't create base block"); //TODO What if tryCreate fails due to a duplicate key? We should ensure we don't use duplicate keys.
_baseBlock = std::move(*newBase);
} else {
(*_baseBlock)->write(_data.data(), 0, _data.size());
}
_dataChanged = false;
}
}
void NewBlock::remove() {
if (_baseBlock != none) {
_blockStore->removeFromBaseStore(std::move(*_baseBlock));
}
_dataChanged = false;
}
void NewBlock::flush() {
writeToBaseBlockIfChanged();
ASSERT(_baseBlock != none, "At this point, the base block should already have been created but wasn't");
(*_baseBlock)->flush();
}
size_t NewBlock::size() const {
return _data.size();
}
void NewBlock::resize(size_t newSize) {
_data = cpputils::DataUtils::resize(std::move(_data), newSize);
_dataChanged = true;
}
bool NewBlock::alreadyExistsInBaseStore() const {
return _baseBlock != none;
}
}
}

52
src/blockstore/implementations/caching/NewBlock.h Normal file
View File

@ -0,0 +1,52 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
#include "../../interface/BlockStore.h"
#include <cpp-utils/data/Data.h>
#include <cpp-utils/macros.h>
#include <memory>
namespace blockstore {
namespace caching {
class CachingBlockStore;
//TODO Does it make sense to write a general DataBackedBlock that just stores a Data object and maps the block operations to it?
// Can we reuse that object somewhere else?
// Maybe a second abstract class for BlockRefBackedBlock?
// This is a block that was created in CachingBlockStore, but doesn't exist in the base block store yet.
// It only exists in the cache and it is created in the base block store when destructed.
class NewBlock final: public Block {
public:
NewBlock(const Key &key, cpputils::Data data, CachingBlockStore *blockStore);
~NewBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
void remove();
bool alreadyExistsInBaseStore() const;
private:
CachingBlockStore *_blockStore;
cpputils::Data _data;
boost::optional<cpputils::unique_ref<Block>> _baseBlock;
bool _dataChanged;
void writeToBaseBlockIfChanged();
DISALLOW_COPY_AND_ASSIGN(NewBlock);
};
}
}
#endif

1
src/blockstore/implementations/caching/cache/Cache.cpp vendored Normal file
View File

@ -0,0 +1 @@
#include "Cache.h"

178
src/blockstore/implementations/caching/cache/Cache.h vendored Normal file
View File

@ -0,0 +1,178 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHE_H_
#include "CacheEntry.h"
#include "QueueMap.h"
#include "PeriodicTask.h"
#include <memory>
#include <boost/optional.hpp>
#include <future>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/lock/MutexPoolLock.h>
#include <cpp-utils/pointer/gcc_4_8_compatibility.h>
namespace blockstore {
namespace caching {
template<class Key, class Value, uint32_t MAX_ENTRIES>
class Cache final {
public:
//TODO Current MAX_LIFETIME_SEC only considers time since the element was last pushed to the Cache. Also insert a real MAX_LIFETIME_SEC that forces resync of entries that have been pushed/popped often (e.g. the root blob)
//TODO Experiment with good values
static constexpr double PURGE_LIFETIME_SEC = 0.5; //When an entry has this age, it will be purged from the cache
static constexpr double PURGE_INTERVAL = 0.5; // With this interval, we check for entries to purge
static constexpr double MAX_LIFETIME_SEC = PURGE_LIFETIME_SEC + PURGE_INTERVAL; // This is the oldest age an entry can reach (given purging works in an ideal world, i.e. with the ideal interval and in zero time)
Cache();
~Cache();
uint32_t size() const;
void push(const Key &key, Value value);
boost::optional<Value> pop(const Key &key);
void flush();
private:
void _makeSpaceForEntry(std::unique_lock<std::mutex> *lock);
void _deleteEntry(std::unique_lock<std::mutex> *lock);
void _deleteOldEntriesParallel();
void _deleteAllEntriesParallel();
void _deleteMatchingEntriesAtBeginningParallel(std::function<bool (const CacheEntry<Key, Value> &)> matches);
void _deleteMatchingEntriesAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches);
bool _deleteMatchingEntryAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches);
mutable std::mutex _mutex;
cpputils::LockPool<Key> _currentlyFlushingEntries;
QueueMap<Key, CacheEntry<Key, Value>> _cachedBlocks;
std::unique_ptr<PeriodicTask> _timeoutFlusher;
DISALLOW_COPY_AND_ASSIGN(Cache);
};
template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Key, Value, MAX_ENTRIES>::PURGE_LIFETIME_SEC;
template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Key, Value, MAX_ENTRIES>::PURGE_INTERVAL;
template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Key, Value, MAX_ENTRIES>::MAX_LIFETIME_SEC;
template<class Key, class Value, uint32_t MAX_ENTRIES>
Cache<Key, Value, MAX_ENTRIES>::Cache(): _mutex(), _currentlyFlushingEntries(), _cachedBlocks(), _timeoutFlusher(nullptr) {
//Don't initialize timeoutFlusher in the initializer list,
//because it then might already call Cache::popOldEntries() before Cache is done constructing.
_timeoutFlusher = std::make_unique<PeriodicTask>(std::bind(&Cache::_deleteOldEntriesParallel, this), PURGE_INTERVAL);
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
Cache<Key, Value, MAX_ENTRIES>::~Cache() {
_deleteAllEntriesParallel();
ASSERT(_cachedBlocks.size() == 0, "Error in _deleteAllEntriesParallel()");
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
boost::optional<Value> Cache<Key, Value, MAX_ENTRIES>::pop(const Key &key) {
std::unique_lock<std::mutex> lock(_mutex);
cpputils::MutexPoolLock<Key> lockEntryFromBeingPopped(&_currentlyFlushingEntries, key, &lock);
auto found = _cachedBlocks.pop(key);
if (!found) {
return boost::none;
}
return found->releaseValue();
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::push(const Key &key, Value value) {
std::unique_lock<std::mutex> lock(_mutex);
ASSERT(_cachedBlocks.size() <= MAX_ENTRIES, "Cache too full");
_makeSpaceForEntry(&lock);
_cachedBlocks.push(key, CacheEntry<Key, Value>(std::move(value)));
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_makeSpaceForEntry(std::unique_lock<std::mutex> *lock) {
// _deleteEntry releases the lock while the Value destructor is running.
// So we can destruct multiple entries in parallel and also call pop() or push() while doing so.
// However, if another thread calls push() before we get the lock back, the cache is full again.
// That's why we need the while() loop here.
while (_cachedBlocks.size() == MAX_ENTRIES) {
_deleteEntry(lock);
}
ASSERT(_cachedBlocks.size() < MAX_ENTRIES, "Removing entry from cache didn't work");
};
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_deleteEntry(std::unique_lock<std::mutex> *lock) {
auto key = _cachedBlocks.peekKey();
ASSERT(key != boost::none, "There was no entry to delete");
cpputils::MutexPoolLock<Key> lockEntryFromBeingPopped(&_currentlyFlushingEntries, *key);
auto value = _cachedBlocks.pop();
// Call destructor outside of the unique_lock,
// i.e. pop() and push() can be called here, except for pop() on the element in _currentlyFlushingEntries
lock->unlock();
value = boost::none; // Call destructor
lock->lock();
};
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_deleteAllEntriesParallel() {
return _deleteMatchingEntriesAtBeginningParallel([] (const CacheEntry<Key, Value> &) {
return true;
});
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_deleteOldEntriesParallel() {
return _deleteMatchingEntriesAtBeginningParallel([] (const CacheEntry<Key, Value> &entry) {
return entry.ageSeconds() > PURGE_LIFETIME_SEC;
});
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_deleteMatchingEntriesAtBeginningParallel(std::function<bool (const CacheEntry<Key, Value> &)> matches) {
// Twice the number of cores, so we use full CPU even if half the threads are doing I/O
unsigned int numThreads = 2 * std::max(1u, std::thread::hardware_concurrency());
std::vector<std::future<void>> waitHandles;
for (unsigned int i = 0; i < numThreads; ++i) {
waitHandles.push_back(std::async(std::launch::async, [this, matches] {
_deleteMatchingEntriesAtBeginning(matches);
}));
}
for (auto & waitHandle : waitHandles) {
waitHandle.wait();
}
};
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_deleteMatchingEntriesAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches) {
while (_deleteMatchingEntryAtBeginning(matches)) {}
}
template<class Key, class Value, uint32_t MAX_ENTRIES>
bool Cache<Key, Value, MAX_ENTRIES>::_deleteMatchingEntryAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches) {
// This function can be called in parallel by multiple threads and will then cause the Value destructors
// to be called in parallel. The call to _deleteEntry() releases the lock while the Value destructor is running.
std::unique_lock<std::mutex> lock(_mutex);
if (_cachedBlocks.size() > 0 && matches(*_cachedBlocks.peek())) {
_deleteEntry(&lock);
return true;
} else {
return false;
}
};
template<class Key, class Value, uint32_t MAX_ENTRIES>
uint32_t Cache<Key, Value, MAX_ENTRIES>::size() const {
std::unique_lock<std::mutex> lock(_mutex);
return _cachedBlocks.size();
};
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::flush() {
//TODO Test flush()
return _deleteAllEntriesParallel();
};
}
}
#endif

1
src/blockstore/implementations/caching/cache/CacheEntry.cpp vendored Normal file
View File

@ -0,0 +1 @@
#include "CacheEntry.h"

44
src/blockstore/implementations/caching/cache/CacheEntry.h vendored Normal file
View File

@ -0,0 +1,44 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHEENTRY_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHEENTRY_H_
#include <ctime>
#include <memory>
#include <cpp-utils/macros.h>
#include <boost/date_time/posix_time/posix_time_types.hpp>
namespace blockstore {
namespace caching {
template<class Key, class Value>
class CacheEntry final {
public:
explicit CacheEntry(Value value): _lastAccess(currentTime()), _value(std::move(value)) {
}
CacheEntry(CacheEntry &&) = default;
double ageSeconds() const {
return ((double)(currentTime() - _lastAccess).total_nanoseconds()) / ((double)1000000000);
}
Value releaseValue() {
return std::move(_value);
}
private:
boost::posix_time::ptime _lastAccess;
Value _value;
static boost::posix_time::ptime currentTime() {
return boost::posix_time::microsec_clock::local_time();
}
DISALLOW_COPY_AND_ASSIGN(CacheEntry);
};
}
}
#endif

27
src/blockstore/implementations/caching/cache/PeriodicTask.cpp vendored Normal file
View File

@ -0,0 +1,27 @@
#include "PeriodicTask.h"
#include <cpp-utils/logging/logging.h>
using std::function;
using std::endl;
using namespace cpputils::logging;
namespace blockstore {
namespace caching {
PeriodicTask::PeriodicTask(function<void ()> task, double intervalSec) :
_task(task),
_interval((uint64_t)(UINT64_C(1000000000) * intervalSec)),
_thread(std::bind(&PeriodicTask::_loopIteration, this)) {
_thread.start();
}
bool PeriodicTask::_loopIteration() {
//Has to be boost::this_thread::sleep_for and not std::this_thread::sleep_for, because it has to be interruptible.
//LoopThread will interrupt this method if it has to be restarted.
boost::this_thread::sleep_for(_interval);
_task();
return true; // Run another iteration (don't terminate thread)
}
}
}

32
src/blockstore/implementations/caching/cache/PeriodicTask.h vendored Normal file
View File

@ -0,0 +1,32 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_PERIODICTASK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_PERIODICTASK_H_
#include <functional>
#include <cpp-utils/thread/LoopThread.h>
#include <boost/chrono.hpp>
namespace blockstore {
namespace caching {
class PeriodicTask final {
public:
PeriodicTask(std::function<void ()> task, double intervalSec);
private:
bool _loopIteration();
std::function<void()> _task;
boost::chrono::nanoseconds _interval;
//This member has to be last, so the thread is destructed first. Otherwise the thread might access elements from a
//partly destructed PeriodicTask.
cpputils::LoopThread _thread;
DISALLOW_COPY_AND_ASSIGN(PeriodicTask);
};
}
}
#endif

1
src/blockstore/implementations/caching/cache/QueueMap.cpp vendored Normal file
View File

@ -0,0 +1 @@
#include "QueueMap.h"

121
src/blockstore/implementations/caching/cache/QueueMap.h vendored Normal file
View File

@ -0,0 +1,121 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_QUEUEMAP_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_QUEUEMAP_H_
#include <memory>
#include <unordered_map>
#include <cassert>
#include <boost/optional.hpp>
#include <cpp-utils/macros.h>
#include <cpp-utils/assert/assert.h>
namespace blockstore {
namespace caching {
//TODO FreeList for performance (malloc is expensive)
//TODO Single linked list with pointer to last element (for insertion) should be enough for a queue. No double linked list needed.
// But then, popping arbitrary elements needs to be rewritten so that _removeFromQueue() is _removeSuccessorFromQueue()
// and the map doesn't store the element itself, but its predecessor. That is, popping might be a bit slower. Test with experiments!
// A class that is a queue and a map at the same time. We could also see it as an addressable queue.
template<class Key, class Value>
class QueueMap final {
public:
QueueMap(): _entries(), _sentinel(&_sentinel, &_sentinel) {
}
~QueueMap() {
for (auto &entry : _entries) {
entry.second.release();
}
}
void push(const Key &key, Value value) {
auto newEntry = _entries.emplace(std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple(_sentinel.prev, &_sentinel));
if (!newEntry.second) {
throw std::logic_error("There is already an element with this key");
}
newEntry.first->second.init(&newEntry.first->first, std::move(value));
//The following is ok, because std::unordered_map never invalidates pointers to its entries
_sentinel.prev->next = &newEntry.first->second;
_sentinel.prev = &newEntry.first->second;
}
boost::optional<Value> pop(const Key &key) {
auto found = _entries.find(key);
if (found == _entries.end()) {
return boost::none;
}
_removeFromQueue(found->second);
auto value = found->second.release();
_entries.erase(found);
return std::move(value);
}
boost::optional<Value> pop() {
if(_sentinel.next == &_sentinel) {
return boost::none;
}
return pop(*_sentinel.next->key);
}
boost::optional<const Key &> peekKey() {
if(_sentinel.next == &_sentinel) {
return boost::none;
}
return *_sentinel.next->key;
}
boost::optional<const Value &> peek() {
if(_sentinel.next == &_sentinel) {
return boost::none;
}
return _sentinel.next->value();
}
uint32_t size() const {
return _entries.size();
}
private:
class Entry final {
public:
Entry(Entry *prev_, Entry *next_): prev(prev_), next(next_), key(nullptr), __value() {
}
void init(const Key *key_, Value value_) {
key = key_;
new(__value) Value(std::move(value_));
}
Value release() {
Value value = std::move(*_value());
_value()->~Value();
return value;
}
const Value &value() {
return *_value();
}
Entry *prev;
Entry *next;
const Key *key;
private:
Value *_value() {
return reinterpret_cast<Value*>(__value);
}
alignas(Value) char __value[sizeof(Value)];
DISALLOW_COPY_AND_ASSIGN(Entry);
};
void _removeFromQueue(const Entry &entry) {
entry.prev->next = entry.next;
entry.next->prev = entry.prev;
}
std::unordered_map<Key, Entry> _entries;
Entry _sentinel;
DISALLOW_COPY_AND_ASSIGN(QueueMap);
};
}
}
#endif

1
src/blockstore/implementations/compressing/CompressedBlock.cpp Normal file
View File

@ -0,0 +1 @@
#include "CompressedBlock.h"

127
src/blockstore/implementations/compressing/CompressedBlock.h Normal file
View File

@ -0,0 +1,127 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSEDBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSEDBLOCK_H_
#include "../../interface/Block.h"
#include "../../interface/BlockStore.h"
#include <cpp-utils/data/DataUtils.h>
#include <cpp-utils/pointer/unique_ref.h>
#include <mutex>
namespace blockstore {
class BlockStore;
namespace compressing {
template<class Compressor> class CompressingBlockStore;
template<class Compressor>
class CompressedBlock final: public Block {
public:
static boost::optional<cpputils::unique_ref<CompressedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData);
static cpputils::unique_ref<CompressedBlock> Decompress(cpputils::unique_ref<Block> baseBlock);
CompressedBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data decompressedData);
~CompressedBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref<Block> releaseBaseBlock();
private:
void _compressToBaseBlock();
cpputils::unique_ref<Block> _baseBlock;
cpputils::Data _decompressedData;
std::mutex _mutex;
bool _dataChanged;
DISALLOW_COPY_AND_ASSIGN(CompressedBlock);
};
template<class Compressor>
boost::optional<cpputils::unique_ref<CompressedBlock<Compressor>>> CompressedBlock<Compressor>::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData) {
cpputils::Data compressed = Compressor::Compress(decompressedData);
auto baseBlock = baseBlockStore->tryCreate(key, std::move(compressed));
if (baseBlock == boost::none) {
//TODO Test this code branch
return boost::none;
}
return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(*baseBlock), std::move(decompressedData));
}
template<class Compressor>
cpputils::unique_ref<CompressedBlock<Compressor>> CompressedBlock<Compressor>::Decompress(cpputils::unique_ref<Block> baseBlock) {
cpputils::Data decompressed = Compressor::Decompress((byte*)baseBlock->data(), baseBlock->size());
return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(baseBlock), std::move(decompressed));
}
template<class Compressor>
CompressedBlock<Compressor>::CompressedBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data decompressedData)
: Block(baseBlock->key()),
_baseBlock(std::move(baseBlock)),
_decompressedData(std::move(decompressedData)),
_dataChanged(false) {
}
template<class Compressor>
CompressedBlock<Compressor>::~CompressedBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_compressToBaseBlock();
}
template<class Compressor>
const void *CompressedBlock<Compressor>::data() const {
return _decompressedData.data();
}
template<class Compressor>
void CompressedBlock<Compressor>::write(const void *source, uint64_t offset, uint64_t size) {
std::memcpy((uint8_t*)_decompressedData.dataOffset(offset), source, size);
_dataChanged = true;
}
template<class Compressor>
void CompressedBlock<Compressor>::flush() {
std::unique_lock<std::mutex> lock(_mutex);
_compressToBaseBlock();
return _baseBlock->flush();
}
template<class Compressor>
size_t CompressedBlock<Compressor>::size() const {
return _decompressedData.size();
}
template<class Compressor>
void CompressedBlock<Compressor>::resize(size_t newSize) {
_decompressedData = cpputils::DataUtils::resize(std::move(_decompressedData), newSize);
_dataChanged = true;
}
template<class Compressor>
cpputils::unique_ref<Block> CompressedBlock<Compressor>::releaseBaseBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_compressToBaseBlock();
return std::move(_baseBlock);
}
template<class Compressor>
void CompressedBlock<Compressor>::_compressToBaseBlock() {
if (_dataChanged) {
cpputils::Data compressed = Compressor::Compress(_decompressedData);
_baseBlock->resize(compressed.size());
_baseBlock->write(compressed.data(), 0, compressed.size());
_dataChanged = false;
}
}
}
}
#endif

1
src/blockstore/implementations/compressing/CompressingBlockStore.cpp Normal file
View File

@ -0,0 +1 @@
#include "CompressingBlockStore.h"

77
src/blockstore/implementations/compressing/CompressingBlockStore.h Normal file
View File

@ -0,0 +1,77 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSINGBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSINGBLOCKSTORE_H_
#include "../../interface/BlockStore.h"
#include "CompressedBlock.h"
namespace blockstore {
namespace compressing {
template<class Compressor>
class CompressingBlockStore final: public BlockStore {
public:
CompressingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
~CompressingBlockStore();
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
DISALLOW_COPY_AND_ASSIGN(CompressingBlockStore);
};
template<class Compressor>
CompressingBlockStore<Compressor>::CompressingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore)
: _baseBlockStore(std::move(baseBlockStore)) {
}
template<class Compressor>
CompressingBlockStore<Compressor>::~CompressingBlockStore() {
}
template<class Compressor>
Key CompressingBlockStore<Compressor>::createKey() {
return _baseBlockStore->createKey();
}
template<class Compressor>
boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::tryCreate(const Key &key, cpputils::Data data) {
auto result = CompressedBlock<Compressor>::TryCreateNew(_baseBlockStore.get(), key, std::move(data));
if (result == boost::none) {
return boost::none;
}
return cpputils::unique_ref<Block>(std::move(*result));
}
template<class Compressor>
boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::load(const Key &key) {
auto loaded = _baseBlockStore->load(key);
if (loaded == boost::none) {
return boost::none;
}
return boost::optional<cpputils::unique_ref<Block>>(CompressedBlock<Compressor>::Decompress(std::move(*loaded)));
}
template<class Compressor>
void CompressingBlockStore<Compressor>::remove(cpputils::unique_ref<Block> block) {
auto _block = cpputils::dynamic_pointer_move<CompressedBlock<Compressor>>(block);
ASSERT(_block != boost::none, "Wrong block type");
auto baseBlock = (*_block)->releaseBaseBlock();
return _baseBlockStore->remove(std::move(baseBlock));
}
template<class Compressor>
uint64_t CompressingBlockStore<Compressor>::numBlocks() const {
return _baseBlockStore->numBlocks();
}
}
}
#endif

29
src/blockstore/implementations/compressing/compressors/Gzip.cpp Normal file
View File

@ -0,0 +1,29 @@
#include "Gzip.h"
#include <cryptopp/gzip.h>
using cpputils::Data;
namespace blockstore {
namespace compressing {
Data Gzip::Compress(const Data &data) {
CryptoPP::Gzip zipper;
zipper.Put((byte *) data.data(), data.size());
zipper.MessageEnd();
Data compressed(zipper.MaxRetrievable());
zipper.Get((byte *) compressed.data(), compressed.size());
return compressed;
}
Data Gzip::Decompress(const void *data, size_t size) {
//TODO Change interface to taking cpputils::Data objects (needs changing blockstore so we can read their "class Data", because this is called from CompressedBlock::Decompress()).
CryptoPP::Gunzip zipper;
zipper.Put((byte *) data, size);
zipper.MessageEnd();
Data decompressed(zipper.MaxRetrievable());
zipper.Get((byte *) decompressed.data(), decompressed.size());
return decompressed;
}
}
}

18
src/blockstore/implementations/compressing/compressors/Gzip.h Normal file
View File

@ -0,0 +1,18 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_GZIP_H
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_GZIP_H
#include <cpp-utils/data/Data.h>
namespace blockstore {
namespace compressing {
class Gzip {
public:
static cpputils::Data Compress(const cpputils::Data &data);
static cpputils::Data Decompress(const void *data, size_t size);
};
}
}
#endif

139
src/blockstore/implementations/compressing/compressors/RunLengthEncoding.cpp Normal file
View File

@ -0,0 +1,139 @@
#include "RunLengthEncoding.h"
#include <sstream>
#include <cpp-utils/assert/assert.h>
using cpputils::Data;
using std::string;
using std::ostringstream;
using std::istringstream;
namespace blockstore {
namespace compressing {
// Alternatively store a run of arbitrary bytes and a run of identical bytes.
// Each run is preceded by its length. Length fields are uint16_t.
// Example: 2 - 5 - 8 - 10 - 3 - 0 - 2 - 0
// Length 2 arbitrary bytes (values: 5, 8), the next 10 bytes store "3" each,
// then 0 arbitrary bytes and 2x "0".
Data RunLengthEncoding::Compress(const Data &data) {
ostringstream compressed;
uint8_t *current = (uint8_t*)data.data();
uint8_t *end = (uint8_t*)data.data()+data.size();
while (current < end) {
_encodeArbitraryWords(&current, end, &compressed);
ASSERT(current <= end, "Overflow");
if (current == end) {
break;
}
_encodeIdenticalWords(&current, end, &compressed);
ASSERT(current <= end, "Overflow");
}
return _extractData(&compressed);
}
void RunLengthEncoding::_encodeArbitraryWords(uint8_t **current, uint8_t* end, ostringstream *output) {
uint16_t size = _arbitraryRunLength(*current, end);
output->write((const char*)&size, sizeof(uint16_t));
output->write((const char*)*current, size);
*current += size;
}
uint16_t RunLengthEncoding::_arbitraryRunLength(uint8_t *start, uint8_t* end) {
// Each stopping of an arbitrary bytes run costs us 5 byte, because we have to store the length
// for the identical bytes run (2 byte), the identical byte itself (1 byte) and the length for the next arbitrary bytes run (2 byte).
// So to get an advantage from stopping an arbitrary bytes run, at least 6 bytes have to be identical.
// realEnd avoids an overflow of the 16bit counter
uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max());
// Count the number of identical bytes and return if it finds a run of more than 6 identical bytes.
uint8_t lastByte = *start + 1; // Something different from the first byte
uint8_t numIdenticalBytes = 1;
for(uint8_t *current = start; current != realEnd; ++current) {
if (*current == lastByte) {
++numIdenticalBytes;
if (numIdenticalBytes == 6) {
return current - start - 5; //-5, because the end pointer for the arbitrary byte run should point to the first identical byte, not the one before.
}
} else {
numIdenticalBytes = 1;
}
lastByte = *current;
}
//It wasn't worth stopping the arbitrary bytes run anywhere. The whole region should be an arbitrary run.
return realEnd-start;
}
void RunLengthEncoding::_encodeIdenticalWords(uint8_t **current, uint8_t* end, ostringstream *output) {
uint16_t size = _countIdenticalBytes(*current, end);
output->write((const char*)&size, sizeof(uint16_t));
output->write((const char*)*current, 1);
*current += size;
}
uint16_t RunLengthEncoding::_countIdenticalBytes(uint8_t *start, uint8_t *end) {
uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max()); // This prevents overflow of the 16bit counter
for (uint8_t *current = start+1; current != realEnd; ++current) {
if (*current != *start) {
return current-start;
}
}
// All bytes have been identical
return realEnd - start;
}
Data RunLengthEncoding::_extractData(ostringstream *stream) {
string str = stream->str();
Data data(str.size());
std::memcpy(data.data(), str.c_str(), str.size());
return data;
}
Data RunLengthEncoding::Decompress(const void *data, size_t size) {
istringstream stream;
_parseData((uint8_t*)data, size, &stream);
ostringstream decompressed;
while(_hasData(&stream)) {
_decodeArbitraryWords(&stream, &decompressed);
if (!_hasData(&stream)) {
break;
}
_decodeIdenticalWords(&stream, &decompressed);
}
return _extractData(&decompressed);
}
bool RunLengthEncoding::_hasData(istringstream *str) {
str->peek();
return !str->eof();
}
void RunLengthEncoding::_parseData(const uint8_t *data, size_t size, istringstream *result) {
result->str(string((const char*)data, size));
}
void RunLengthEncoding::_decodeArbitraryWords(istringstream *stream, ostringstream *decompressed) {
uint16_t size;
stream->read((char*)&size, sizeof(uint16_t));
ASSERT(stream->good(), "Premature end of stream");
Data run(size);
stream->read((char*)run.data(), size);
ASSERT(stream->good(), "Premature end of stream");
decompressed->write((const char*)run.data(), run.size());
}
void RunLengthEncoding::_decodeIdenticalWords(istringstream *stream, ostringstream *decompressed) {
uint16_t size;
stream->read((char*)&size, sizeof(uint16_t));
ASSERT(stream->good(), "Premature end of stream");
uint8_t value;
stream->read((char*)&value, 1);
ASSERT(stream->good(), "Premature end of stream");
Data run(size);
std::memset(run.data(), value, run.size());
decompressed->write((const char*)run.data(), run.size());
}
}
}

29
src/blockstore/implementations/compressing/compressors/RunLengthEncoding.h Normal file
View File

@ -0,0 +1,29 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_RUNLENGTHENCODING_H
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_RUNLENGTHENCODING_H
#include <cpp-utils/data/Data.h>
namespace blockstore {
namespace compressing {
class RunLengthEncoding {
public:
static cpputils::Data Compress(const cpputils::Data &data);
static cpputils::Data Decompress(const void *data, size_t size);
private:
static void _encodeArbitraryWords(uint8_t **current, uint8_t* end, std::ostringstream *output);
static uint16_t _arbitraryRunLength(uint8_t *start, uint8_t* end);
static void _encodeIdenticalWords(uint8_t **current, uint8_t* end, std::ostringstream *output);
static uint16_t _countIdenticalBytes(uint8_t *start, uint8_t *end);
static bool _hasData(std::istringstream *stream);
static cpputils::Data _extractData(std::ostringstream *stream);
static void _parseData(const uint8_t *data, size_t size, std::istringstream *result);
static void _decodeArbitraryWords(std::istringstream *stream, std::ostringstream *decompressed);
static void _decodeIdenticalWords(std::istringstream *stream, std::ostringstream *decompressed);
};
}
}
#endif

1
src/blockstore/implementations/encrypted/EncryptedBlock.cpp Normal file
View File

@ -0,0 +1 @@
#include "EncryptedBlock.h"

177
src/blockstore/implementations/encrypted/EncryptedBlock.h Normal file
View File

@ -0,0 +1,177 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include "../../interface/BlockStore.h"
#include <cpp-utils/macros.h>
#include <memory>
#include <iostream>
#include <boost/optional.hpp>
#include <cpp-utils/crypto/symmetric/Cipher.h>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
#include <mutex>
#include <cpp-utils/logging/logging.h>
namespace blockstore {
namespace encrypted {
template<class Cipher> class EncryptedBlockStore;
//TODO Test EncryptedBlock
//TODO Fix mutexes & locks (basically true for all blockstores)
template<class Cipher>
class EncryptedBlock final: public Block {
public:
BOOST_CONCEPT_ASSERT((cpputils::CipherConcept<Cipher>));
static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey);
static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key);
//TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key, cpputils::Data plaintextWithHeader);
~EncryptedBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t count) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref<Block> releaseBlock();
private:
cpputils::unique_ref<Block> _baseBlock; // TODO Do I need the ciphertext block in memory or is the key enough?
cpputils::Data _plaintextWithHeader;
typename Cipher::EncryptionKey _encKey;
bool _dataChanged;
static constexpr unsigned int HEADER_LENGTH = Key::BINARY_LENGTH;
void _encryptToBaseBlock();
static cpputils::Data _prependKeyHeaderToData(const Key &key, cpputils::Data data);
static bool _keyHeaderIsCorrect(const Key &key, const cpputils::Data &data);
std::mutex _mutex;
DISALLOW_COPY_AND_ASSIGN(EncryptedBlock);
};
template<class Cipher>
constexpr unsigned int EncryptedBlock<Cipher>::HEADER_LENGTH;
template<class Cipher>
boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey) {
cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, std::move(data));
cpputils::Data encrypted = Cipher::encrypt((byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), encKey);
auto baseBlock = baseBlockStore->tryCreate(key, std::move(encrypted));
if (baseBlock == boost::none) {
//TODO Test this code branch
return boost::none;
}
return cpputils::make_unique_ref<EncryptedBlock>(std::move(*baseBlock), encKey, std::move(plaintextWithHeader));
}
template<class Cipher>
boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey) {
//TODO Change BlockStore so we can read their "class Data" objects instead of "void *data()", and then we can change the Cipher interface to take Data objects instead of "byte *" + size
boost::optional<cpputils::Data> plaintextWithHeader = Cipher::decrypt((byte*)baseBlock->data(), baseBlock->size(), encKey);
if(plaintextWithHeader == boost::none) {
//Decryption failed (e.g. an authenticated cipher detected modifications to the ciphertext)
cpputils::logging::LOG(cpputils::logging::WARN) << "Decrypting block " << baseBlock->key().ToString() << " failed. Was the block modified by an attacker?";
return boost::none;
}
if(!_keyHeaderIsCorrect(baseBlock->key(), *plaintextWithHeader)) {
//The stored key in the block data is incorrect - an attacker might have exchanged the contents with the encrypted data from a different block
cpputils::logging::LOG(cpputils::logging::WARN) << "Decrypting block " << baseBlock->key().ToString() << " failed due to invalid block key. Was the block modified by an attacker?";
return boost::none;
}
return cpputils::make_unique_ref<EncryptedBlock<Cipher>>(std::move(baseBlock), encKey, std::move(*plaintextWithHeader));
}
template<class Cipher>
cpputils::Data EncryptedBlock<Cipher>::_prependKeyHeaderToData(const Key &key, cpputils::Data data) {
static_assert(HEADER_LENGTH >= Key::BINARY_LENGTH, "Key doesn't fit into the header");
cpputils::Data result(data.size() + HEADER_LENGTH);
std::memcpy(result.data(), key.data(), Key::BINARY_LENGTH);
std::memcpy((uint8_t*)result.data() + Key::BINARY_LENGTH, data.data(), data.size());
return result;
}
template<class Cipher>
bool EncryptedBlock<Cipher>::_keyHeaderIsCorrect(const Key &key, const cpputils::Data &data) {
return 0 == std::memcmp(key.data(), data.data(), Key::BINARY_LENGTH);
}
template<class Cipher>
EncryptedBlock<Cipher>::EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey, cpputils::Data plaintextWithHeader)
:Block(baseBlock->key()),
_baseBlock(std::move(baseBlock)),
_plaintextWithHeader(std::move(plaintextWithHeader)),
_encKey(encKey),
_dataChanged(false),
_mutex() {
}
template<class Cipher>
EncryptedBlock<Cipher>::~EncryptedBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
}
template<class Cipher>
const void *EncryptedBlock<Cipher>::data() const {
return (uint8_t*)_plaintextWithHeader.data() + HEADER_LENGTH;
}
template<class Cipher>
void EncryptedBlock<Cipher>::write(const void *source, uint64_t offset, uint64_t count) {
ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition
std::memcpy((uint8_t*)_plaintextWithHeader.data()+HEADER_LENGTH+offset, source, count);
_dataChanged = true;
}
template<class Cipher>
void EncryptedBlock<Cipher>::flush() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
return _baseBlock->flush();
}
template<class Cipher>
size_t EncryptedBlock<Cipher>::size() const {
return _plaintextWithHeader.size() - HEADER_LENGTH;
}
template<class Cipher>
void EncryptedBlock<Cipher>::resize(size_t newSize) {
_plaintextWithHeader = cpputils::DataUtils::resize(std::move(_plaintextWithHeader), newSize + HEADER_LENGTH);
_dataChanged = true;
}
template<class Cipher>
void EncryptedBlock<Cipher>::_encryptToBaseBlock() {
if (_dataChanged) {
cpputils::Data encrypted = Cipher::encrypt((byte*)_plaintextWithHeader.data(), _plaintextWithHeader.size(), _encKey);
_baseBlock->write(encrypted.data(), 0, encrypted.size());
_dataChanged = false;
}
}
template<class Cipher>
cpputils::unique_ref<Block> EncryptedBlock<Cipher>::releaseBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
return std::move(_baseBlock);
}
}
}
#endif

1
src/blockstore/implementations/encrypted/EncryptedBlockStore.cpp Normal file
View File

@ -0,0 +1 @@
#include "EncryptedBlockStore.h"

90
src/blockstore/implementations/encrypted/EncryptedBlockStore.h Normal file
View File

@ -0,0 +1,90 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
#include "../../interface/BlockStore.h"
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/cast.h>
#include "EncryptedBlock.h"
#include <iostream>
namespace blockstore {
namespace encrypted {
template<class Cipher>
class EncryptedBlockStore final: public BlockStore {
public:
EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey);
//TODO Are createKey() tests included in generic BlockStoreTest? If not, add it!
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
//This function should only be used by test cases
void __setKey(const typename Cipher::EncryptionKey &encKey);
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
typename Cipher::EncryptionKey _encKey;
DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore);
};
template<class Cipher>
EncryptedBlockStore<Cipher>::EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
: _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
}
template<class Cipher>
Key EncryptedBlockStore<Cipher>::createKey() {
return _baseBlockStore->createKey();
}
template<class Cipher>
boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::tryCreate(const Key &key, cpputils::Data data) {
//TODO Test that this returns boost::none when base blockstore returns nullptr (for all pass-through-blockstores)
//TODO Easier implementation? This is only so complicated because of the case EncryptedBlock -> Block
auto result = EncryptedBlock<Cipher>::TryCreateNew(_baseBlockStore.get(), key, std::move(data), _encKey);
if (result == boost::none) {
return boost::none;
}
return cpputils::unique_ref<Block>(std::move(*result));
}
template<class Cipher>
boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::load(const Key &key) {
auto block = _baseBlockStore->load(key);
if (block == boost::none) {
//TODO Test this path (for all pass-through-blockstores)
return boost::none;
}
return boost::optional<cpputils::unique_ref<Block>>(EncryptedBlock<Cipher>::TryDecrypt(std::move(*block), _encKey));
}
template<class Cipher>
void EncryptedBlockStore<Cipher>::remove(cpputils::unique_ref<Block> block) {
auto encryptedBlock = cpputils::dynamic_pointer_move<EncryptedBlock<Cipher>>(block);
ASSERT(encryptedBlock != boost::none, "Block is not an EncryptedBlock");
auto baseBlock = (*encryptedBlock)->releaseBlock();
return _baseBlockStore->remove(std::move(baseBlock));
}
template<class Cipher>
uint64_t EncryptedBlockStore<Cipher>::numBlocks() const {
return _baseBlockStore->numBlocks();
}
template<class Cipher>
void EncryptedBlockStore<Cipher>::__setKey(const typename Cipher::EncryptionKey &encKey) {
_encKey = encKey;
}
}
}
#endif

50
src/blockstore/implementations/inmemory/InMemoryBlock.cpp Normal file
View File

@ -0,0 +1,50 @@
#include "InMemoryBlock.h"
#include "InMemoryBlockStore.h"
#include <cstring>
#include <cpp-utils/data/DataUtils.h>
#include <cpp-utils/assert/assert.h>
using std::make_shared;
using std::istream;
using std::ostream;
using std::ifstream;
using std::ofstream;
using std::ios;
using cpputils::Data;
namespace blockstore {
namespace inmemory {
InMemoryBlock::InMemoryBlock(const Key &key, Data data)
: Block(key), _data(make_shared<Data>(std::move(data))) {
}
InMemoryBlock::InMemoryBlock(const InMemoryBlock &rhs)
: Block(rhs), _data(rhs._data) {
}
InMemoryBlock::~InMemoryBlock() {
}
const void *InMemoryBlock::data() const {
return _data->data();
}
void InMemoryBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data->size() && offset + size <= _data->size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
std::memcpy((uint8_t*)_data->data()+offset, source, size);
}
size_t InMemoryBlock::size() const {
return _data->size();
}
void InMemoryBlock::resize(size_t newSize) {
*_data = cpputils::DataUtils::resize(std::move(*_data), newSize);
}
void InMemoryBlock::flush() {
}
}
}

33
src/blockstore/implementations/inmemory/InMemoryBlock.h Normal file
View File

@ -0,0 +1,33 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
namespace blockstore {
namespace inmemory {
class InMemoryBlockStore;
class InMemoryBlock final: public Block {
public:
InMemoryBlock(const Key &key, cpputils::Data size);
InMemoryBlock(const InMemoryBlock &rhs);
~InMemoryBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
private:
std::shared_ptr<cpputils::Data> _data;
};
}
}
#endif

56
src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp Normal file
View File

@ -0,0 +1,56 @@
#include "InMemoryBlock.h"
#include "InMemoryBlockStore.h"
#include <memory>
#include <cpp-utils/assert/assert.h>
using std::make_unique;
using std::string;
using std::mutex;
using std::lock_guard;
using std::piecewise_construct;
using std::make_tuple;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
namespace blockstore {
namespace inmemory {
InMemoryBlockStore::InMemoryBlockStore()
: _blocks() {}
optional<unique_ref<Block>> InMemoryBlockStore::tryCreate(const Key &key, Data data) {
auto insert_result = _blocks.emplace(piecewise_construct, make_tuple(key.ToString()), make_tuple(key, std::move(data)));
if (!insert_result.second) {
return none;
}
//Return a pointer to the stored InMemoryBlock
return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(insert_result.first->second));
}
optional<unique_ref<Block>> InMemoryBlockStore::load(const Key &key) {
//Return a pointer to the stored InMemoryBlock
try {
return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(_blocks.at(key.ToString())));
} catch (const std::out_of_range &e) {
return none;
}
}
void InMemoryBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
cpputils::destruct(std::move(block));
int numRemoved = _blocks.erase(key.ToString());
ASSERT(1==numRemoved, "Didn't find block to remove");
}
uint64_t InMemoryBlockStore::numBlocks() const {
return _blocks.size();
}
}
}

33
src/blockstore/implementations/inmemory/InMemoryBlockStore.h Normal file
View File

@ -0,0 +1,33 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
#include <cpp-utils/macros.h>
#include <mutex>
#include <map>
namespace blockstore {
namespace inmemory {
class InMemoryBlock;
class InMemoryBlockStore final: public BlockStoreWithRandomKeys {
public:
InMemoryBlockStore();
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
private:
std::map<std::string, InMemoryBlock> _blocks;
DISALLOW_COPY_AND_ASSIGN(InMemoryBlockStore);
};
}
}
#endif

108
src/blockstore/implementations/ondisk/OnDiskBlock.cpp Normal file
View File

@ -0,0 +1,108 @@
#include <cstring>
#include <fstream>
#include <boost/filesystem.hpp>
#include "OnDiskBlock.h"
#include "OnDiskBlockStore.h"
#include "../../utils/FileDoesntExistException.h"
#include <cpp-utils/data/DataUtils.h>
#include <cpp-utils/assert/assert.h>
using std::istream;
using std::ostream;
using std::ifstream;
using std::ofstream;
using std::ios;
using cpputils::Data;
using cpputils::make_unique_ref;
using cpputils::unique_ref;
using boost::optional;
using boost::none;
namespace bf = boost::filesystem;
namespace blockstore {
namespace ondisk {
OnDiskBlock::OnDiskBlock(const Key &key, const bf::path &filepath, Data data)
: Block(key), _filepath(filepath), _data(std::move(data)), _dataChanged(false), _mutex() {
}
OnDiskBlock::~OnDiskBlock() {
flush();
}
const void *OnDiskBlock::data() const {
return _data.data();
}
void OnDiskBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
std::memcpy((uint8_t*)_data.data()+offset, source, size);
_dataChanged = true;
}
size_t OnDiskBlock::size() const {
return _data.size();
}
void OnDiskBlock::resize(size_t newSize) {
_data = cpputils::DataUtils::resize(std::move(_data), newSize);
_dataChanged = true;
}
optional<unique_ref<OnDiskBlock>> OnDiskBlock::LoadFromDisk(const bf::path &rootdir, const Key &key) {
auto filepath = rootdir / key.ToString();
try {
//If it isn't a file, Data::LoadFromFile() would usually also crash. We still need this extra check
//upfront, because Data::LoadFromFile() doesn't crash if we give it the path of a directory
//instead the path of a file.
//TODO Data::LoadFromFile now returns boost::optional. Do we then still need this?
if(!bf::is_regular_file(filepath)) {
return none;
}
boost::optional<Data> data = Data::LoadFromFile(filepath);
if (!data) {
return none;
}
return make_unique_ref<OnDiskBlock>(key, filepath, std::move(*data));
} catch (const FileDoesntExistException &e) {
return none;
}
}
optional<unique_ref<OnDiskBlock>> OnDiskBlock::CreateOnDisk(const bf::path &rootdir, const Key &key, Data data) {
auto filepath = rootdir / key.ToString();
if (bf::exists(filepath)) {
return none;
}
auto block = make_unique_ref<OnDiskBlock>(key, filepath, std::move(data));
block->_storeToDisk();
return std::move(block);
}
void OnDiskBlock::RemoveFromDisk(const bf::path &rootdir, const Key &key) {
auto filepath = rootdir / key.ToString();
ASSERT(bf::is_regular_file(filepath), "Block not found on disk");
bf::remove(filepath);
}
void OnDiskBlock::_fillDataWithZeroes() {
_data.FillWithZeroes();
_dataChanged = true;
}
void OnDiskBlock::_storeToDisk() const {
_data.StoreToFile(_filepath);
}
void OnDiskBlock::flush() {
std::unique_lock<std::mutex> lock(_mutex);
if (_dataChanged) {
_storeToDisk();
_dataChanged = false;
}
}
}
}

50
src/blockstore/implementations/ondisk/OnDiskBlock.h Normal file
View File

@ -0,0 +1,50 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
#include <boost/filesystem/path.hpp>
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include <iostream>
#include <cpp-utils/pointer/unique_ref.h>
#include <mutex>
namespace blockstore {
namespace ondisk {
class OnDiskBlockStore;
class OnDiskBlock final: public Block {
public:
OnDiskBlock(const Key &key, const boost::filesystem::path &filepath, cpputils::Data data);
~OnDiskBlock();
static boost::optional<cpputils::unique_ref<OnDiskBlock>> LoadFromDisk(const boost::filesystem::path &rootdir, const Key &key);
static boost::optional<cpputils::unique_ref<OnDiskBlock>> CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data);
static void RemoveFromDisk(const boost::filesystem::path &rootdir, const Key &key);
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
private:
const boost::filesystem::path _filepath;
cpputils::Data _data;
bool _dataChanged;
void _fillDataWithZeroes();
void _storeToDisk() const;
std::mutex _mutex;
DISALLOW_COPY_AND_ASSIGN(OnDiskBlock);
};
}
}
#endif

52
src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp Normal file
View File

@ -0,0 +1,52 @@
#include "OnDiskBlock.h"
#include "OnDiskBlockStore.h"
using std::string;
using cpputils::Data;
using cpputils::unique_ref;
using boost::optional;
using boost::none;
namespace bf = boost::filesystem;
namespace blockstore {
namespace ondisk {
OnDiskBlockStore::OnDiskBlockStore(const boost::filesystem::path &rootdir)
: _rootdir(rootdir) {
if (!bf::exists(rootdir)) {
throw std::runtime_error("Base directory not found");
}
if (!bf::is_directory(rootdir)) {
throw std::runtime_error("Base directory is not a directory");
}
//TODO Test for read access, write access, enter (x) access, and throw runtime_error in case
}
//TODO Do I have to lock tryCreate/remove and/or load? Or does ParallelAccessBlockStore take care of that?
optional<unique_ref<Block>> OnDiskBlockStore::tryCreate(const Key &key, Data data) {
//TODO Easier implementation? This is only so complicated because of the cast OnDiskBlock -> Block
auto result = std::move(OnDiskBlock::CreateOnDisk(_rootdir, key, std::move(data)));
if (result == boost::none) {
return boost::none;
}
return unique_ref<Block>(std::move(*result));
}
optional<unique_ref<Block>> OnDiskBlockStore::load(const Key &key) {
return optional<unique_ref<Block>>(OnDiskBlock::LoadFromDisk(_rootdir, key));
}
void OnDiskBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
cpputils::destruct(std::move(block));
OnDiskBlock::RemoveFromDisk(_rootdir, key);
}
uint64_t OnDiskBlockStore::numBlocks() const {
return std::distance(bf::directory_iterator(_rootdir), bf::directory_iterator());
}
}
}

31
src/blockstore/implementations/ondisk/OnDiskBlockStore.h Normal file
View File

@ -0,0 +1,31 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
#include <boost/filesystem.hpp>
#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
#include <cpp-utils/macros.h>
namespace blockstore {
namespace ondisk {
class OnDiskBlockStore final: public BlockStoreWithRandomKeys {
public:
explicit OnDiskBlockStore(const boost::filesystem::path &rootdir);
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
private:
const boost::filesystem::path _rootdir;
DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore);
};
}
}
#endif

1
src/blockstore/implementations/parallelaccess/BlockRef.cpp Normal file
View File

@ -0,0 +1 @@
#include "BlockRef.h"

48
src/blockstore/implementations/parallelaccess/BlockRef.h Normal file
View File

@ -0,0 +1,48 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_BLOCKREF_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_BLOCKREF_H_
#include <parallelaccessstore/ParallelAccessStore.h>
#include "../../interface/Block.h"
#include <cpp-utils/macros.h>
#include <memory>
namespace blockstore {
namespace parallelaccess {
class ParallelAccessBlockStore;
class BlockRef final: public Block, public parallelaccessstore::ParallelAccessStore<Block, BlockRef, Key>::ResourceRefBase {
public:
//TODO Unneccessarily storing Key twice here (in parent class and in _baseBlock).
explicit BlockRef(Block *baseBlock): Block(baseBlock->key()), _baseBlock(baseBlock) {}
const void *data() const override {
return _baseBlock->data();
}
void write(const void *source, uint64_t offset, uint64_t size) override {
return _baseBlock->write(source, offset, size);
}
void flush() override {
return _baseBlock->flush();
}
size_t size() const override {
return _baseBlock->size();
}
void resize(size_t newSize) override {
return _baseBlock->resize(newSize);
}
private:
Block *_baseBlock;
DISALLOW_COPY_AND_ASSIGN(BlockRef);
};
}
}
#endif

60
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.cpp Normal file
View File

@ -0,0 +1,60 @@
#include "BlockRef.h"
#include "ParallelAccessBlockStore.h"
#include "ParallelAccessBlockStoreAdapter.h"
#include <cassert>
#include <cpp-utils/pointer/cast.h>
#include <cpp-utils/assert/assert.h>
using std::string;
using std::promise;
using cpputils::dynamic_pointer_move;
using cpputils::make_unique_ref;
using boost::none;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
namespace blockstore {
namespace parallelaccess {
ParallelAccessBlockStore::ParallelAccessBlockStore(unique_ref<BlockStore> baseBlockStore)
: _baseBlockStore(std::move(baseBlockStore)), _parallelAccessStore(make_unique_ref<ParallelAccessBlockStoreAdapter>(_baseBlockStore.get())) {
}
Key ParallelAccessBlockStore::createKey() {
return _baseBlockStore->createKey();
}
optional<unique_ref<Block>> ParallelAccessBlockStore::tryCreate(const Key &key, cpputils::Data data) {
ASSERT(!_parallelAccessStore.isOpened(key), ("Key "+key.ToString()+"already exists").c_str());
auto block = _baseBlockStore->tryCreate(key, std::move(data));
if (block == none) {
//TODO Test this code branch
return none;
}
return unique_ref<Block>(_parallelAccessStore.add(key, std::move(*block)));
}
optional<unique_ref<Block>> ParallelAccessBlockStore::load(const Key &key) {
auto block = _parallelAccessStore.load(key);
if (block == none) {
return none;
}
return unique_ref<Block>(std::move(*block));
}
void ParallelAccessBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
auto block_ref = dynamic_pointer_move<BlockRef>(block);
ASSERT(block_ref != none, "Block is not a BlockRef");
return _parallelAccessStore.remove(key, std::move(*block_ref));
}
uint64_t ParallelAccessBlockStore::numBlocks() const {
return _baseBlockStore->numBlocks();
}
}
}

34
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h Normal file
View File

@ -0,0 +1,34 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTORE_H_
#include <parallelaccessstore/ParallelAccessStore.h>
#include "BlockRef.h"
#include "../../interface/BlockStore.h"
#include <cpp-utils/pointer/unique_ref.h>
namespace blockstore {
namespace parallelaccess {
//TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel)
class ParallelAccessBlockStore final: public BlockStore {
public:
explicit ParallelAccessBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
parallelaccessstore::ParallelAccessStore<Block, BlockRef, Key> _parallelAccessStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessBlockStore);
};
}
}
#endif

1
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp Normal file
View File

@ -0,0 +1 @@
#include "ParallelAccessBlockStoreAdapter.h"

35
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.h Normal file
View File

@ -0,0 +1,35 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTOREADAPTER_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTOREADAPTER_H_
#include <cpp-utils/macros.h>
#include <parallelaccessstore/ParallelAccessStore.h>
#include "../../interface/BlockStore.h"
namespace blockstore {
namespace parallelaccess {
class ParallelAccessBlockStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore<Block, Key> {
public:
explicit ParallelAccessBlockStoreAdapter(BlockStore *baseBlockStore)
:_baseBlockStore(std::move(baseBlockStore)) {
}
boost::optional<cpputils::unique_ref<Block>> loadFromBaseStore(const Key &key) override {
return _baseBlockStore->load(key);
}
void removeFromBaseStore(cpputils::unique_ref<Block> block) override {
return _baseBlockStore->remove(std::move(block));
}
private:
BlockStore *_baseBlockStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessBlockStoreAdapter);
};
}
}
#endif

54
src/blockstore/implementations/testfake/FakeBlock.cpp Normal file
View File

@ -0,0 +1,54 @@
#include "FakeBlock.h"
#include "FakeBlockStore.h"
#include <cstring>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
using std::shared_ptr;
using std::istream;
using std::ostream;
using std::ifstream;
using std::ofstream;
using std::ios;
using std::string;
using cpputils::Data;
namespace blockstore {
namespace testfake {
FakeBlock::FakeBlock(FakeBlockStore *store, const Key &key, shared_ptr<Data> data, bool dirty)
: Block(key), _store(store), _data(data), _dataChanged(dirty) {
}
FakeBlock::~FakeBlock() {
flush();
}
const void *FakeBlock::data() const {
return _data->data();
}
void FakeBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data->size() && offset + size <= _data->size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
std::memcpy((uint8_t*)_data->data()+offset, source, size);
_dataChanged = true;
}
size_t FakeBlock::size() const {
return _data->size();
}
void FakeBlock::resize(size_t newSize) {
*_data = cpputils::DataUtils::resize(std::move(*_data), newSize);
_dataChanged = true;
}
void FakeBlock::flush() {
if(_dataChanged) {
_store->updateData(key(), *_data);
_dataChanged = false;
}
}
}
}

39
src/blockstore/implementations/testfake/FakeBlock.h Normal file
View File

@ -0,0 +1,39 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include <cpp-utils/macros.h>
namespace blockstore {
namespace testfake {
class FakeBlockStore;
class FakeBlock final: public Block {
public:
FakeBlock(FakeBlockStore *store, const Key &key, std::shared_ptr<cpputils::Data> data, bool dirty);
~FakeBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
private:
FakeBlockStore *_store;
std::shared_ptr<cpputils::Data> _data;
bool _dataChanged;
DISALLOW_COPY_AND_ASSIGN(FakeBlock);
};
}
}
#endif

80
src/blockstore/implementations/testfake/FakeBlockStore.cpp Normal file
View File

@ -0,0 +1,80 @@
#include "FakeBlock.h"
#include "FakeBlockStore.h"
#include <cpp-utils/assert/assert.h>
using std::make_shared;
using std::string;
using std::mutex;
using std::lock_guard;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
namespace blockstore {
namespace testfake {
FakeBlockStore::FakeBlockStore()
: _blocks(), _used_dataregions_for_blocks(), _mutex() {}
optional<unique_ref<Block>> FakeBlockStore::tryCreate(const Key &key, Data data) {
std::unique_lock<std::mutex> lock(_mutex);
auto insert_result = _blocks.emplace(key.ToString(), std::move(data));
if (!insert_result.second) {
return none;
}
//Return a copy of the stored data
return _load(key);
}
optional<unique_ref<Block>> FakeBlockStore::load(const Key &key) {
std::unique_lock<std::mutex> lock(_mutex);
return _load(key);
}
optional<unique_ref<Block>> FakeBlockStore::_load(const Key &key) {
//Return a copy of the stored data
string key_string = key.ToString();
try {
return makeFakeBlockFromData(key, _blocks.at(key_string), false);
} catch (const std::out_of_range &e) {
return none;
}
}
void FakeBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
cpputils::destruct(std::move(block));
std::unique_lock<std::mutex> lock(_mutex);
int numRemoved = _blocks.erase(key.ToString());
ASSERT(numRemoved == 1, "Block not found");
}
unique_ref<Block> FakeBlockStore::makeFakeBlockFromData(const Key &key, const Data &data, bool dirty) {
auto newdata = make_shared<Data>(data.copy());
_used_dataregions_for_blocks.push_back(newdata);
return make_unique_ref<FakeBlock>(this, key, newdata, dirty);
}
void FakeBlockStore::updateData(const Key &key, const Data &data) {
std::unique_lock<std::mutex> lock(_mutex);
auto found = _blocks.find(key.ToString());
if (found == _blocks.end()) {
auto insertResult = _blocks.emplace(key.ToString(), data.copy());
ASSERT(true == insertResult.second, "Inserting didn't work");
found = insertResult.first;
}
Data &stored_data = found->second;
stored_data = data.copy();
}
uint64_t FakeBlockStore::numBlocks() const {
std::unique_lock<std::mutex> lock(_mutex);
return _blocks.size();
}
}
}

62
src/blockstore/implementations/testfake/FakeBlockStore.h Normal file
View File

@ -0,0 +1,62 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCKSTORE_H_
#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
#include <cpp-utils/data/Data.h>
#include <cpp-utils/macros.h>
#include <mutex>
#include <map>
namespace blockstore {
namespace testfake {
class FakeBlock;
/**
* This blockstore is meant to be used for unit tests when the module under test needs a blockstore to work with.
* It basically is the same as the InMemoryBlockStore, but much less forgiving for programming mistakes.
*
* InMemoryBlockStore for example simply ignores flushing and gives you access to the same data region each time
* you request a block. This is very performant, but also forgiving to mistakes. Say you write over the boundaries
* of a block, then you wouldn't notice, since the next time you access the block, the overflow data is (probably)
* still there. Or say an application is relying on flushing the block store in the right moment. Since flushing
* is a no-op in InMemoryBlockStore, you wouldn't notice either.
*
* So this FakeBlockStore has a background copy of each block. When you request a block, you will get a copy of
* the data (instead of a direct pointer as InMemoryBlockStore does) and flushing will copy the data back to the
* background. This way, tests are more likely to fail if they use the blockstore wrongly.
*/
class FakeBlockStore final: public BlockStoreWithRandomKeys {
public:
FakeBlockStore();
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
void updateData(const Key &key, const cpputils::Data &data);
private:
std::map<std::string, cpputils::Data> _blocks;
//This vector keeps a handle of the data regions for all created FakeBlock objects.
//This way, it is ensured that no two created FakeBlock objects will work on the
//same data region. Without this, it could happen that a test case creates a FakeBlock,
//destructs it, creates another one, and the new one gets the same memory region.
//We want to avoid this for the reasons mentioned above (overflow data).
std::vector<std::shared_ptr<cpputils::Data>> _used_dataregions_for_blocks;
mutable std::mutex _mutex;
cpputils::unique_ref<Block> makeFakeBlockFromData(const Key &key, const cpputils::Data &data, bool dirty);
boost::optional<cpputils::unique_ref<Block>> _load(const Key &key);
DISALLOW_COPY_AND_ASSIGN(FakeBlockStore);
};
}
}
#endif

41
src/blockstore/interface/Block.h Normal file
View File

@ -0,0 +1,41 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_INTERFACE_BLOCK_H_
#define MESSMER_BLOCKSTORE_INTERFACE_BLOCK_H_
#include "../utils/Key.h"
#include <cstring>
namespace blockstore {
//TODO Make Block non-virtual class that stores ptr to its blockstore and writes itself back to the blockstore who is offering a corresponding function.
// Then ondisk blockstore can be actually create the file on disk in blockstore::create() and cachingblockstore will delay that call to its base block store.
class Block {
public:
virtual ~Block() {}
virtual const void *data() const = 0;
virtual void write(const void *source, uint64_t offset, uint64_t size) = 0;
virtual void flush() = 0;
virtual size_t size() const = 0;
//TODO Test resize()
virtual void resize(size_t newSize) = 0;
const Key &key() const {
return _key;
}
protected:
Block(const Key &key) : _key(key) {}
private:
const Key _key;
};
}
#endif

39
src/blockstore/interface/BlockStore.h Normal file
View File

@ -0,0 +1,39 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_INTERFACE_BLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_INTERFACE_BLOCKSTORE_H_
#include "Block.h"
#include <string>
#include <boost/optional.hpp>
#include <cpp-utils/pointer/unique_ref.h>
#include <cpp-utils/data/Data.h>
namespace blockstore {
class BlockStore {
public:
virtual ~BlockStore() {}
virtual Key createKey() = 0;
//Returns boost::none if key already exists
virtual boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) = 0;
//TODO Use boost::optional (if key doesn't exist)
// Return nullptr if block with this key doesn't exists
virtual boost::optional<cpputils::unique_ref<Block>> load(const Key &key) = 0;
virtual void remove(cpputils::unique_ref<Block> block) = 0;
virtual uint64_t numBlocks() const = 0;
cpputils::unique_ref<Block> create(const cpputils::Data &data) {
while(true) {
//TODO Copy (data.copy()) necessary?
auto block = tryCreate(createKey(), data.copy());
if (block != boost::none) {
return std::move(*block);
}
}
}
};
}
#endif

1
src/blockstore/interface/helpers/BlockStoreWithRandomKeys.cpp Normal file
View File

@ -0,0 +1 @@
#include "BlockStoreWithRandomKeys.h"

23
src/blockstore/interface/helpers/BlockStoreWithRandomKeys.h Normal file
View File

@ -0,0 +1,23 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_INTERFACE_HELPERS_BLOCKSTOREWITHRANDOMKEYS_H_
#define MESSMER_BLOCKSTORE_INTERFACE_HELPERS_BLOCKSTOREWITHRANDOMKEYS_H_
#include "../BlockStore.h"
#include "../Block.h"
#include <cpp-utils/random/Random.h>
namespace blockstore {
// This is an implementation helpers for BlockStores that use random block keys.
// You should never give this static type to the client. The client should always
// work with the BlockStore interface instead.
class BlockStoreWithRandomKeys: public BlockStore {
public:
Key createKey() final {
return cpputils::Random::PseudoRandom().getFixedSize<Key::BINARY_LENGTH>();
}
};
}
#endif

31
src/blockstore/utils/BlockStoreUtils.cpp Normal file
View File

@ -0,0 +1,31 @@
#include "../interface/BlockStore.h"
#include "BlockStoreUtils.h"
#include <cpp-utils/data/Data.h>
#include <cassert>
#include <cpp-utils/assert/assert.h>
using cpputils::Data;
using cpputils::unique_ref;
namespace blockstore {
namespace utils {
unique_ref<Block> copyToNewBlock(BlockStore *blockStore, const Block &block) {
Data data(block.size());
std::memcpy(data.data(), block.data(), block.size());
return blockStore->create(data);
}
void copyTo(Block *target, const Block &source) {
ASSERT(target->size() == source.size(), "Can't copy block data when blocks have different sizes");
target->write(source.data(), 0, source.size());
}
void fillWithZeroes(Block *target) {
Data zeroes(target->size());
zeroes.FillWithZeroes();
target->write(zeroes.data(), 0, target->size());
}
}
}

19
src/blockstore/utils/BlockStoreUtils.h Normal file
View File

@ -0,0 +1,19 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_UTILS_BLOCKSTOREUTILS_H_
#define MESSMER_BLOCKSTORE_UTILS_BLOCKSTOREUTILS_H_
#include <cpp-utils/pointer/unique_ref.h>
namespace blockstore {
class BlockStore;
class Block;
namespace utils {
cpputils::unique_ref<Block> copyToNewBlock(BlockStore *blockStore, const Block &block);
void copyTo(Block *target, const Block &source);
void fillWithZeroes(Block *target);
}
}
#endif

17
src/blockstore/utils/FileDoesntExistException.cpp Normal file
View File

@ -0,0 +1,17 @@
#include "FileDoesntExistException.h"
namespace bf = boost::filesystem;
using std::runtime_error;
using std::string;
namespace blockstore {
FileDoesntExistException::FileDoesntExistException(const bf::path &filepath)
: runtime_error(string("The file ")+filepath.c_str()+" doesn't exist") {
}
FileDoesntExistException::~FileDoesntExistException() {
}
}

19
src/blockstore/utils/FileDoesntExistException.h Normal file
View File

@ -0,0 +1,19 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_UTILS_FILEDOESNTEXISTEXCEPTION_H_
#define MESSMER_BLOCKSTORE_UTILS_FILEDOESNTEXISTEXCEPTION_H_
#include <boost/filesystem/path.hpp>
#include <stdexcept>
namespace blockstore {
class FileDoesntExistException final: public std::runtime_error {
public:
explicit FileDoesntExistException(const boost::filesystem::path &filepath);
~FileDoesntExistException();
};
}
#endif

1
src/blockstore/utils/Key.cpp Normal file
View File

@ -0,0 +1 @@
#include "Key.h"

32
src/blockstore/utils/Key.h Normal file
View File

@ -0,0 +1,32 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_UTILS_KEY_H_
#define MESSMER_BLOCKSTORE_UTILS_KEY_H_
#include <string>
#include <cpp-utils/data/FixedSizeData.h>
namespace blockstore {
// A key here is NOT a key for encryption, but a key as used in key->value mappings ("access handle for a block").
//TODO Rename to BlockId/BlobId and make it a class containing a FixedSizeData<> member
using Key = cpputils::FixedSizeData<16>;
}
namespace std {
//Allow using blockstore::Key in std::unordered_map / std::unordered_set
template <> struct hash<blockstore::Key> {
size_t operator()(const blockstore::Key &key) const {
//Keys are random, so it is enough to use the first few bytes as a hash
return *(size_t*)(key.data());
}
};
//Allow using blockstore::Key in std::map / std::set
template <> struct less<blockstore::Key> {
bool operator()(const blockstore::Key &lhs, const blockstore::Key &rhs) const {
return 0 > std::memcmp(lhs.data(), rhs.data(), blockstore::Key::BINARY_LENGTH);
}
};
}
#endif

51
src/cpp-utils/CMakeLists.txt Normal file
View File

@ -0,0 +1,51 @@
project (cpp-utils)
set(SOURCES
crypto/symmetric/ciphers.cpp
crypto/kdf/DerivedKey.cpp
crypto/kdf/Scrypt.cpp
crypto/kdf/DerivedKeyConfig.cpp
crypto/RandomPadding.cpp
process/daemonize.cpp
process/subprocess.cpp
tempfile/TempFile.cpp
tempfile/TempDir.cpp
network/HttpClient.cpp
network/CurlHttpClient.cpp
network/FakeHttpClient.cpp
io/Console.cpp
io/pipestream.cpp
thread/LoopThread.cpp
thread/ThreadSystem.cpp
random/Random.cpp
random/RandomGeneratorThread.cpp
random/OSRandomGenerator.cpp
random/PseudoRandomPool.cpp
random/RandomDataBuffer.cpp
random/RandomGenerator.cpp
lock/LockPool.cpp
data/Serializer.cpp
data/Deserializer.cpp
data/DataFixture.cpp
data/DataUtils.cpp
data/Data.cpp
assert/backtrace.cpp
assert/AssertFailed.cpp
)
add_library(${PROJECT_NAME} STATIC ${SOURCES})
# This is needed by boost thread
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PRIVATE rt)
endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
target_link_libraries(${PROJECT_NAME} PRIVATE pthread curl)
# TODO From Crypto++ 5.7 on, it should support cmake with find_package().
find_library(CryptoPP cryptopp)
target_link_libraries(${PROJECT_NAME} PUBLIC ${CryptoPP} scrypt spdlog)
target_add_boost(${PROJECT_NAME} filesystem system thread)
target_enable_style_warnings(${PROJECT_NAME})
target_activate_cpp14(${PROJECT_NAME})

1
src/cpp-utils/assert/AssertFailed.cpp Normal file
View File

@ -0,0 +1 @@
#include "AssertFailed.h"

25
src/cpp-utils/assert/AssertFailed.h Normal file
View File

@ -0,0 +1,25 @@
#pragma once
#ifndef MESSMER_CPPUTILS_ASSERT_ASSERTFAILED_H
#define MESSMER_CPPUTILS_ASSERT_ASSERTFAILED_H
#include <stdexcept>
#include <string>
#include "../macros.h"
namespace cpputils {
class AssertFailed final: public std::exception {
public:
AssertFailed(const std::string &message) : _message(message) { }
const char *what() const throw() override {
return _message.c_str();
}
private:
std::string _message;
};
}
#endif

45
src/cpp-utils/assert/assert.h Normal file
View File

@ -0,0 +1,45 @@
#pragma once
#ifndef MESSMER_CPPUTILS_ASSERT_ASSERT_H
#define MESSMER_CPPUTILS_ASSERT_ASSERT_H
/**
* This implements an ASSERT(expr, msg) macro.
* In a debug build, it will crash and halt the program on an assert failure.
* In a release build, it will throw an AssertFailed exception instead, which can then be caught.
*/
#include "AssertFailed.h"
#include <iostream>
#include "backtrace.h"
#include "../logging/logging.h"
namespace cpputils {
namespace _assert {
inline std::string format(const char *expr, const std::string &message, const char *file, int line) {
std::string result = std::string()+"Assertion ["+expr+"] failed in "+file+":"+std::to_string(line)+": "+message+"\n\n" + backtrace();
return result;
}
inline void assert_fail_release [[noreturn]] (const char *expr, const std::string &message, const char *file, int line) {
auto msg = format(expr, message, file, line);
using namespace logging;
LOG(ERROR) << msg;
throw AssertFailed(msg);
}
inline void assert_fail_debug [[noreturn]] (const char *expr, const std::string &message, const char *file, int line) {
using namespace logging;
LOG(ERROR) << format(expr, message, file, line);
abort();
}
}
}
#ifdef NDEBUG
//TODO Check whether disabling assertions in prod affects speed.
# define ASSERT(expr, msg) (void)((expr) || (cpputils::_assert::assert_fail_release(#expr, msg, __FILE__, __LINE__),0))
#else
# define ASSERT(expr, msg) (void)((expr) || (cpputils::_assert::assert_fail_debug(#expr, msg, __FILE__, __LINE__),0))
#endif
#endif

68
src/cpp-utils/assert/backtrace.cpp Normal file
View File

@ -0,0 +1,68 @@
#include "backtrace.h"
#include <execinfo.h>
#include <signal.h>
#include <iostream>
#include <unistd.h>
#include <cxxabi.h>
#include <string>
#include <sstream>
#include "../logging/logging.h"
using std::string;
using std::ostringstream;
using namespace cpputils::logging;
//TODO Use the following? https://github.com/bombela/backward-cpp
namespace cpputils {
//TODO Refactor (for example: RAII or at least try{}finally{} instead of free())
std::string demangle(const string &mangledName) {
string result;
int status = -10;
char *demangledName = abi::__cxa_demangle(mangledName.c_str(), NULL, NULL, &status);
if (status == 0) {
result = demangledName;
} else {
result = mangledName;
}
free(demangledName);
return result;
}
std::string pretty(const string &backtraceLine) {
size_t startMangledName = backtraceLine.find('(');
size_t endMangledName = backtraceLine.find('+');
if (startMangledName == string::npos || endMangledName == string::npos) {
return backtraceLine;
}
return demangle(backtraceLine.substr(startMangledName+1, endMangledName-startMangledName-1)) + ": (" + backtraceLine.substr(0, startMangledName) + backtraceLine.substr(endMangledName);
}
string backtrace_to_string(void *array[], size_t size) {
ostringstream result;
char **ptr = backtrace_symbols(array, size);
for (size_t i = 0; i < size; ++i) {
result << pretty(ptr[i]) << "\n";
}
free(ptr);
return result.str();
}
string backtrace() {
constexpr unsigned int MAX_SIZE = 100;
void *array[MAX_SIZE];
size_t size = ::backtrace(array, MAX_SIZE);
return backtrace_to_string(array, size);
}
void sigsegv_handler(int) {
LOG(ERROR) << "SIGSEGV\n" << backtrace();
exit(1);
}
void showBacktraceOnSigSegv() {
signal(SIGSEGV, sigsegv_handler);
}
}

12
src/cpp-utils/assert/backtrace.h Normal file
View File

@ -0,0 +1,12 @@
#pragma once
#ifndef MESSMER_CPPUTILS_ASSERT_BACKTRACE_H
#define MESSMER_CPPUTILS_ASSERT_BACKTRACE_H
#include <string>
namespace cpputils {
std::string backtrace();
void showBacktraceOnSigSegv();
}
#endif

34
src/cpp-utils/crypto/RandomPadding.cpp Normal file
View File

@ -0,0 +1,34 @@
#include "RandomPadding.h"
#include "../logging/logging.h"
#include "../random/Random.h"
using boost::optional;
using namespace cpputils::logging;
namespace cpputils {
Data RandomPadding::add(const Data &data, size_t targetSize) {
uint32_t size = data.size();
if (size >= targetSize - sizeof(size)) {
throw std::runtime_error("Data too large. We should increase padding target size.");
}
Data randomData = Random::PseudoRandom().get(targetSize-sizeof(size)-size);
ASSERT(sizeof(size) + size + randomData.size() == targetSize, "Calculated size of randomData incorrectly");
Data result(targetSize);
std::memcpy(reinterpret_cast<char*>(result.data()), &size, sizeof(size));
std::memcpy(reinterpret_cast<char*>(result.dataOffset(sizeof(size))), reinterpret_cast<const char*>(data.data()), size);
std::memcpy(reinterpret_cast<char*>(result.dataOffset(sizeof(size)+size)), reinterpret_cast<const char*>(randomData.data()), randomData.size());
return result;
}
optional<Data> RandomPadding::remove(const Data &data) {
uint32_t size;
std::memcpy(&size, reinterpret_cast<const char*>(data.data()), sizeof(size));
if(sizeof(size) + size >= data.size()) {
LOG(ERROR) << "Config file is invalid: Invalid padding.";
return boost::none;
};
Data result(size);
std::memcpy(reinterpret_cast<char*>(result.data()), reinterpret_cast<const char*>(data.dataOffset(sizeof(size))), size);
return std::move(result);
}
}

Some files were not shown because too many files have changed in this diff Show More