Merge from next

.circleci/config.yml

@@ -7,7 +7,7 @@ references:
     run:
       name: Initialize Cache
       command: |
-        echo "${APT_COMPILER_PACKAGE}_${BUILD_TOOLSET}_${CXX}_${CC}" > /tmp/_build_env_vars
+        echo "${APT_COMPILER_PACKAGE}_${BUILD_TOOLSET}_${CXX}_${CC}_${BUILD_TYPE}_${CXXFLAGS}" > /tmp/_build_env_vars
         echo Build env vars used for cache keys:
         cat /tmp/_build_env_vars
   container_setup_pre: &container_setup_pre
@@ -49,7 +49,7 @@ references:
         sudo chmod o-w /etc/apt/sources.list.d/clang.list
 
         DEBIAN_FRONTEND=noninteractive sudo apt-get update -qq
-        DEBIAN_FRONTEND=noninteractive sudo apt-get install -y git ccache $APT_COMPILER_PACKAGE cmake make libcurl4-openssl-dev libboost-filesystem-dev libboost-system-dev libboost-chrono-dev libboost-program-options-dev libboost-thread-dev libcrypto++-dev libssl-dev libfuse-dev python
+        DEBIAN_FRONTEND=noninteractive sudo apt-get install -y git ccache $APT_COMPILER_PACKAGE cmake3 make libcurl4-openssl-dev libssl-dev libfuse-dev python
         # Use /dev/urandom when /dev/random is accessed to use less entropy
         sudo cp -a /dev/urandom /dev/random
 
@@ -150,26 +150,34 @@ references:
         ccache --max-size=512M
         ccache --show-stats
 
+        # Disable OpenMP if it is clang, because Ubuntu 14.04 doesn't have the libomp-dev package needed to support OpenMP for clang.
+        if [[ ${APT_COMPILER_PACKAGE} == clang* ]]; then
+          OPENMP_PARAMS="-DDISABLE_OPENMP=ON"
+        else
+          OPENMP_PARAMS=""
+        fi
+
         # Build
         mkdir cmake
         cd cmake
-        cmake .. -DBUILD_TESTING=on -DCMAKE_BUILD_TYPE=Debug
+        cmake .. -DBUILD_TESTING=on -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${OPENMP_PARAMS}
         make -j$NUMCORES
 
         ccache --show-stats
   test: &test
     run:
       name: Test
+      no_output_timeout: 120m
       command: |
         cd cmake
         ./test/gitversion/gitversion-test
-        ./test/cpp-utils/cpp-utils-test
+        if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] ; then ./test/cpp-utils/cpp-utils-test ; fi
-        ./test/fspp/fspp-test
+        if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] && [ ! "$DISABLE_BROKEN_ASAN_TESTS" = true ] ; then ./test/fspp/fspp-test ; fi
         ./test/parallelaccessstore/parallelaccessstore-test
         ./test/blockstore/blockstore-test
         ./test/blobstore/blobstore-test
-        ./test/cryfs/cryfs-test
+        if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] ; then ./test/cryfs/cryfs-test ; fi
-        ./test/cryfs-cli/cryfs-cli-test
+        if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] ; then ./test/cryfs-cli/cryfs-cli-test ; fi
   job_definition: &job_definition
     <<: *container_config
     steps:
@@ -191,82 +199,156 @@ references:
         only: /.*/
 
 jobs:
-  gcc_4_8:
+  gcc_5_debug:
-    <<: *job_definition
-    environment:
-      CC: gcc-4.8
-      CXX: g++-4.8
-      BUILD_TOOLSET: gcc
-      APT_COMPILER_PACKAGE: "g++-4.8"
-  gcc_5:
     <<: *job_definition
     environment:
       CC: gcc-5
       CXX: g++-5
       BUILD_TOOLSET: gcc
       APT_COMPILER_PACKAGE: "g++-5"
-  gcc_6:
+      CXXFLAGS: ""
+      BUILD_TYPE: "Debug"
+  gcc_5_release:
+    <<: *job_definition
+    environment:
+      CC: gcc-5
+      CXX: g++-5
+      BUILD_TOOLSET: gcc
+      APT_COMPILER_PACKAGE: "g++-5"
+      CXXFLAGS: ""
+      BUILD_TYPE: "Release"
+  gcc_6_debug:
     <<: *job_definition
     environment:
       CC: gcc-6
       CXX: g++-6
       BUILD_TOOLSET: gcc
       APT_COMPILER_PACKAGE: "g++-6"
-  gcc_7:
+      CXXFLAGS: ""
+      BUILD_TYPE: "Debug"
+  gcc_6_release:
+    <<: *job_definition
+    environment:
+      CC: gcc-6
+      CXX: g++-6
+      BUILD_TOOLSET: gcc
+      APT_COMPILER_PACKAGE: "g++-6"
+      CXXFLAGS: ""
+      BUILD_TYPE: "Release"
+  gcc_7_debug:
     <<: *job_definition
     environment:
       CC: gcc-7
       CXX: g++-7
       BUILD_TOOLSET: gcc
       APT_COMPILER_PACKAGE: "g++-7"
-  clang_3_7:
+      CXXFLAGS: ""
+      BUILD_TYPE: "Debug"
+  gcc_7_release:
     <<: *job_definition
     environment:
-      CC: clang-3.7
+      CC: gcc-7
-      CXX: clang++-3.7
+      CXX: g++-7
-      BUILD_TOOLSET: clang
+      BUILD_TOOLSET: gcc
-      APT_COMPILER_PACKAGE: clang-3.7
+      APT_COMPILER_PACKAGE: "g++-7"
-  clang_3_8:
+      CXXFLAGS: ""
-    <<: *job_definition
+      BUILD_TYPE: "Release"
-    environment:
+  clang_4_0_debug:
-      CC: clang-3.8
-      CXX: clang++-3.8
-      BUILD_TOOLSET: clang
-      APT_COMPILER_PACKAGE: clang-3.8
-  clang_4_0:
     <<: *job_definition
     environment:
       CC: clang-4.0
       CXX: clang++-4.0
       BUILD_TOOLSET: clang
       APT_COMPILER_PACKAGE: clang-4.0
-  clang_5_0:
+      CXXFLAGS: ""
+      BUILD_TYPE: "Debug"
+  clang_4_0_release:
+    <<: *job_definition
+    environment:
+      CC: clang-4.0
+      CXX: clang++-4.0
+      BUILD_TOOLSET: clang
+      APT_COMPILER_PACKAGE: clang-4.0
+      CXXFLAGS: ""
+      BUILD_TYPE: "Release"
+  clang_5_0_debug:
     <<: *job_definition
     environment:
       CC: clang-5.0
       CXX: clang++-5.0
       BUILD_TOOLSET: clang
       APT_COMPILER_PACKAGE: clang-5.0
+      CXXFLAGS: ""
+      BUILD_TYPE: "Debug"
+  clang_5_0_release:
+    <<: *job_definition
+    environment:
+      CC: clang-5.0
+      CXX: clang++-5.0
+      BUILD_TOOLSET: clang
+      APT_COMPILER_PACKAGE: clang-5.0
+      CXXFLAGS: ""
+      BUILD_TYPE: "Release"
+  no_compatibility:
+    <<: *job_definition
+    environment:
+      CC: clang-5.0
+      CXX: clang++-5.0
+      BUILD_TOOLSET: clang
+      APT_COMPILER_PACKAGE: clang-5.0
+      CXXFLAGS: "-DCRYFS_NO_COMPATIBILITY"
+      BUILD_TYPE: "Debug"
+  address_sanitizer:
+    <<: *job_definition
+    environment:
+      CC: clang-5.0
+      CXX: clang++-5.0
+      BUILD_TOOLSET: clang
+      APT_COMPILER_PACKAGE: clang-5.0
+      CXXFLAGS: "-O2 -fsanitize=address -fno-omit-frame-pointer -fno-common -fsanitize-address-use-after-scope"
+      BUILD_TYPE: "Debug"
+      # Note: Leak detection is disabled because libfuse itself is leaky...
+      ASAN_OPTIONS: "detect_leaks=0 check_initialization_order=1 detect_stack_use_after_return=1 detect_invalid_pointer_pairs=1 atexit=1"
+      DISABLE_BROKEN_ASAN_TESTS: true
+  thread_sanitizer:
+    <<: *job_definition
+    environment:
+      CC: clang-5.0
+      CXX: clang++-5.0
+      BUILD_TOOLSET: clang
+      APT_COMPILER_PACKAGE: clang-5.0
+      CXXFLAGS: "-O2 -fsanitize=thread -fno-omit-frame-pointer"
+      BUILD_TYPE: "Debug"
+      DISABLE_BROKEN_TSAN_TESTS: true
 
 workflows:
   version: 2
 
   build_and_test:
     jobs:
-    - gcc_4_8:
+    - gcc_5_debug:
         <<: *enable_for_tags
-    - gcc_5:
+    - gcc_5_release:
         <<: *enable_for_tags
-    - gcc_6:
+    - gcc_6_debug:
         <<: *enable_for_tags
-    - gcc_7:
+    - gcc_6_release:
         <<: *enable_for_tags
-    - clang_3_7:
+    - gcc_7_debug:
         <<: *enable_for_tags
-    - clang_3_8:
+    - gcc_7_release:
         <<: *enable_for_tags
-    - clang_4_0:
+    - clang_4_0_debug:
         <<: *enable_for_tags
-    - clang_5_0:
+    - clang_4_0_release:
+        <<: *enable_for_tags
+    - clang_5_0_debug:
+        <<: *enable_for_tags
+    - clang_5_0_release:
+        <<: *enable_for_tags
+    - no_compatibility:
+        <<: *enable_for_tags
+    - address_sanitizer:
+        <<: *enable_for_tags
+    - thread_sanitizer:
         <<: *enable_for_tags
-

.clang-tidy

@@ -0,0 +1,50 @@
+---
+# TODO Enable (some of) the explicitly disabled checks. Possibly needs helper types from gsl library or similar to enable full cppcoreguidelines.
+# TODO Enable more checks (google-*, hicpp-*, llvm-*, modernize-*, mpi-*, performance-*, readability-*)
+# TODO Maybe just enable * and disable a list instead?
+Checks: |
+  clang-diagnostic-*,
+  clang-analyzer-*,
+  bugprone-*,
+  cert-*,
+  cppcoreguidelines-*,
+  misc-*,
+  boost-use-to-string,
+  -cert-env33-c,
+  -cert-err58-cpp,
+  -cppcoreguidelines-owning-memory,
+  -cppcoreguidelines-no-malloc,
+  -cppcoreguidelines-pro-type-const-cast,
+  -cppcoreguidelines-pro-bounds-pointer-arithmetic,
+  -cppcoreguidelines-pro-type-reinterpret-cast,
+  -cppcoreguidelines-special-member-functions,
+  -cppcoreguidelines-pro-type-cstyle-cast,
+  -cppcoreguidelines-pro-bounds-array-to-pointer-decay,
+  -cppcoreguidelines-pro-type-vararg,
+  -misc-macro-parentheses,
+  -misc-unused-raii
+WarningsAsErrors: ''
+HeaderFilterRegex: '/src/|/test/'
+CheckOptions:
+  - key:             google-readability-braces-around-statements.ShortStatementLines
+    value:           '1'
+  - key:             google-readability-function-size.StatementThreshold
+    value:           '800'
+  - key:             google-readability-namespace-comments.ShortNamespaceLines
+    value:           '10'
+  - key:             google-readability-namespace-comments.SpacesBeforeComments
+    value:           '2'
+  - key:             modernize-loop-convert.MaxCopySize
+    value:           '16'
+  - key:             modernize-loop-convert.MinConfidence
+    value:           reasonable
+  - key:             modernize-loop-convert.NamingStyle
+    value:           CamelCase
+  - key:             modernize-pass-by-value.IncludeStyle
+    value:           llvm
+  - key:             modernize-replace-auto-ptr.IncludeStyle
+    value:           llvm
+  - key:             modernize-use-nullptr.NullMacros
+    value:           'NULL'
+...
+

.gitignore

@@ -2,6 +2,7 @@ umltest.inner.sh
 umltest.status
 /build
 /cmake
+/cmake-build-*
 /.idea
 *~
 

.travis.yml

@@ -5,7 +5,7 @@ compiler:
 - gcc
 - clang
 os:
-- linux
+#- linux
 - osx
 addons:
   apt:
@@ -18,34 +18,6 @@ addons:
     - libcrypto++-dev
     - libfuse-dev
 install:
-# Use new clang
+- .travisci/install.sh
-- if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "$CXX" = "clang++" ]; then export CXX="clang++-3.7" CC="clang-3.7"; fi
-# Detect number of CPU cores
-- export NUMCORES=`grep -c ^processor /proc/cpuinfo` && if [ ! -n "$NUMCORES" ]; then export NUMCORES=`sysctl -n hw.ncpu`; fi
-- echo Using $NUMCORES cores
-# Install dependencies
-- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./travis.install_boost.sh; fi
-- if [ "${TRAVIS_OS_NAME}" == "osx" ]; then brew cask install osxfuse && brew install cryptopp; fi
-# Install run_with_fuse.sh
-- mkdir cmake
-- cd cmake
-- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/run_with_fuse.sh
-- chmod +x run_with_fuse.sh
-- cmake --version
-#  Use /dev/urandom when /dev/random is accessed, because travis doesn't have enough entropy
-- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then sudo cp -a /dev/urandom /dev/random; fi
 script:
-- cmake .. -DBUILD_TESTING=on -DCMAKE_BUILD_TYPE=Debug
+- .travisci/build_and_test.sh
-- make -j$NUMCORES
-- ./test/gitversion/gitversion-test
-- ./test/cpp-utils/cpp-utils-test
-# TODO Also run on osx once fixed
-- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./run_with_fuse.sh ./test/fspp/fspp-test || exit 1; fi
-- ./test/parallelaccessstore/parallelaccessstore-test
-- ./test/blockstore/blockstore-test
-- ./test/blobstore/blobstore-test
-# TODO Also run on osx once fixed
-- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./test/cryfs/cryfs-test || exit 1; fi
-- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./test/cryfs-cli/cryfs-cli-test || exit 1; fi
-after_script:
-- rm run_with_fuse.sh

.travisci/build_and_test.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+# Detect number of CPU cores
+export NUMCORES=`grep -c ^processor /proc/cpuinfo`
+if [ ! -n "$NUMCORES" ]; then
+  export NUMCORES=`sysctl -n hw.ncpu`
+fi
+echo Using $NUMCORES cores
+
+# Setup target directory
+mkdir cmake
+cd cmake
+cmake --version
+
+# Build
+cmake .. -DBUILD_TESTING=on -DCMAKE_BUILD_TYPE=Debug
+make -j$NUMCORES
+
+# Test
+./test/gitversion/gitversion-test
+./test/cpp-utils/cpp-utils-test
+./test/parallelaccessstore/parallelaccessstore-test
+./test/blockstore/blockstore-test
+./test/blobstore/blobstore-test
+./test/cryfs/cryfs-test
+
+# TODO Also run on osx once fixed
+if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
+  ./test/fspp/fspp-test
+  ./test/cryfs-cli/cryfs-cli-test
+fi

.travisci/install.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+# Use new clang on linux
+if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "$CXX" = "clang++" ]; then
+  export CXX="clang++-3.7" CC="clang-3.7"
+fi
+
+# If using gcc on mac, actually use it ("gcc" just links to clang, but "gcc-4.8" is gcc, https://github.com/travis-ci/travis-ci/issues/2423)
+if [ "${TRAVIS_OS_NAME}" == "osx" ] && ["${CXX}" = "g++" ]; then
+  export CXX="g++-4.8" CC="gcc-4.8"
+fi
+
+# Install dependencies
+if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
+  ./.travisci/install_boost.sh
+fi
+
+if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+  brew cask install osxfuse
+  brew install libomp
+fi
+
+# By default, travis only fetches the newest 50 commits. We need more in case we're further from the last version tag, so the build doesn't fail because it can't generate the version number.
+git fetch --unshallow
+
+#  Use /dev/urandom when /dev/random is accessed, because travis doesn't have enough entropy
+if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
+  sudo cp -a /dev/urandom /dev/random
+fi

CMakeLists.txt

@@ -1,26 +1,33 @@
-cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+cmake_policy(SET CMP0054 NEW)
+
+# note: for clang-tidy, we need cmake 3.6, or (if the return code should be handled correctly, e.g. on CI), we need 3.8.
+
+# TODO Perf test:
+#  - try if setting CRYPTOPP_NATIVE_ARCH=ON and adding -march=native to the compile commands for cryfs source files makes a difference
+#    -> if yes, offer a cmake option to enable both of these
 
 project(cryfs)
 
 include(utils.cmake)
 
-require_gcc_version(4.8)
+require_gcc_version(5.0)
-require_clang_version(3.7)
+require_clang_version(4.0)
 
 # Default value is not to build test cases
-if(NOT BUILD_TESTING)
+option(BUILD_TESTING "build test cases" OFF)
-    set(BUILD_TESTING OFF CACHE BOOL "BUILD_TESTING")
+option(CRYFS_UPDATE_CHECKS "let cryfs check for updates and security vulnerabilities" ON)
-endif(NOT BUILD_TESTING)
+option(DISABLE_OPENMP "allow building without OpenMP libraries. This will cause performance degradations." OFF)
 
-# Default vaule is to build in release mode
+# Default value is to build in release mode
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release CACHE INTERNAL "CMAKE_BUILD_TYPE")
 endif(NOT CMAKE_BUILD_TYPE)
 
-# Default value is to do update checks
+# The MSVC version on AppVeyor CI needs this
-if(NOT CRYFS_UPDATE_CHECKS)
+if(MSVC)
-    set(CRYFS_UPDATE_CHECKS ON CACHE BOOL "CRYFS_UPDATE_CHECKS")
+  add_definitions(/bigobj)
-endif(NOT CRYFS_UPDATE_CHECKS)
+endif()
 
 add_subdirectory(vendor)
 add_subdirectory(src)

ChangeLog.txt

@@ -1,11 +1,29 @@
+Version 0.10.0 (unreleased)
+---------------
+New Features & Improvements:
+* Integrity checks ensure you notice when someone modifies your file system.
+* File system nodes (files, directories, symlinks) store a parent pointer to the directory that contains them. This information can be used in later versions to resolve some synchronization conflicts.
+* Allow mounting using system mount tool and /etc/fstab (e.g. mount -t fuse.cryfs basedir mountdir)
+* Performance improvements
+* Use relatime instead of strictatime (further performance improvement)
+* Pass fuse options directly to cryfs (i.e. 'cryfs basedir mountdir -o allow_other' instead of 'cryfs basedir mountdir -- -o allow_other')
+* CryFS tells the operating system to not swap the encryption key to the disk (note: this is best-effort and cannot be guaranteed. Hibernation, for example, will still write the encryption key to the disk)
+* New block size options: 4KB and 16KB
+* New default block size: 16KB
+* Increased scrypt hardness to (N=1048576, r=4, p=8) to make it harder to crack the key while allowing cryfs to take advantage of multicore machines.
+
+Fixed bugs:
+* `du` shows correct file system size on Mac OS X.
+
+
 Version 0.9.9
---------------
 Improvements:
 * Add --allow-filesystem-upgrade option which will upgrade old file systems without asking the user. This will be especially helpful for GUI tools.
 * Add --version option that shows the CryFS version and exits.
 * When CryFS fails to load a file system, the process stops with a helpful error code, which can be used by GUI tools to show detailed messages.
 * Only migrate a file system if the underlying storage format changed
 
+
 Version 0.9.8
 --------------
 Compatibility:

README.md

@@ -1,4 +1,5 @@
-# CryFS [![Build Status](https://travis-ci.org/cryfs/cryfs.svg?branch=master)](https://travis-ci.org/cryfs/cryfs)
+# CryFS [![Build Status](https://travis-ci.org/cryfs/cryfs.svg?branch=master)](https://travis-ci.org/cryfs/cryfs) [![CircleCI](https://circleci.com/gh/cryfs/cryfs/tree/master.svg?style=svg)](https://circleci.com/gh/cryfs/cryfs/tree/master)
+
 CryFS encrypts your files, so you can safely store them anywhere. It works well together with cloud services like Dropbox, iCloud, OneDrive and others.
 See [https://www.cryfs.org](https://www.cryfs.org).
 
@@ -22,8 +23,8 @@ Building from source
 Requirements
 ------------
   - Git (for getting the source code)
-  - GCC version >= 4.8 or Clang >= 3.7
+  - GCC version >= 5.0 or Clang >= 4.0
-  - CMake version >= 2.8
+  - CMake version >= 3.0 
   - libcurl4 (including development headers)
   - Boost libraries version >= 1.56 (including development headers)
     - filesystem
@@ -31,21 +32,21 @@ Requirements
     - chrono
     - program_options
     - thread
-  - Crypto++ version >= 5.6.3 (including development headers)
   - SSL development libraries (including development headers, e.g. libssl-dev)
   - libFUSE version >= 2.8.6 (including development headers), on Mac OS X instead install osxfuse from https://osxfuse.github.io/
   - Python >= 2.7
+  - OpenMP
 
 You can use the following commands to install these requirements
 
         # Ubuntu
-        $ sudo apt-get install git g++ cmake make libcurl4-openssl-dev libboost-filesystem-dev libboost-system-dev libboost-chrono-dev libboost-program-options-dev libboost-thread-dev libcrypto++-dev libssl-dev libfuse-dev python
+        $ sudo apt-get install git g++ cmake make libcurl4-openssl-dev libboost-filesystem-dev libboost-system-dev libboost-chrono-dev libboost-program-options-dev libboost-thread-dev libssl-dev libfuse-dev python
 
         # Fedora
-        sudo dnf install git gcc-c++ cmake make libcurl-devel boost-devel boost-static cryptopp-devel openssl-devel fuse-devel python
+        sudo dnf install git gcc-c++ cmake make libcurl-devel boost-devel boost-static openssl-devel fuse-devel python
 
         # Macintosh
-        brew install cmake boost cryptopp openssl
+        brew install cmake boost openssl libomp
 
 Build & Install
 ---------------
@@ -97,13 +98,7 @@ On most systems, CMake should find the libraries automatically. However, that do
 
         cmake .. -DCMAKE_CXX_FLAGS="-I/path/to/fuse/or/osxfuse/headers"
 
-4. **CryptoPP library not found**
+4. **Openssl headers not found**
-
-    Pass in the library path with
-
-        cmake .. -DCRYPTOPP_LIB_PATH=/path/to/cryptopp
-
-5. **Openssl headers not found**
 
     Pass in the include path with
 

TODO-0.10.txt

@@ -0,0 +1 @@
+Change homebrew recipe to "brew install libomp" but not "brew install cryptopp"

doc/CMakeLists.txt

@@ -1,4 +1,8 @@
 project (doc)
+
+IF (WIN32)
+	MESSAGE(STATUS "This is Windows. Will not install man page")
+ELSE (WIN32)
 	INCLUDE(GNUInstallDirs)
 
 	find_program(GZIP gzip)
@@ -13,3 +17,4 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz
 			DESTINATION ${CMAKE_INSTALL_MANDIR}/man1
 			CONFIGURATIONS Release
 	)
+ENDIF(WIN32)

doc/man/cryfs.1

@@ -169,6 +169,24 @@ Allow upgrading the file system if it was created with an old CryFS version. Aft
 .
 .
 .TP
+\fB\-\-allow-integrity-violations\fI
+.
+By default, CryFS checks for integrity violations, i.e. will notice if an adversary modified or rolled back the file system. Using this flag, you can disable the integrity checks. This can for example be helpful for loading an old snapshot of your file system without CryFS thinking an adversary rolled it back.
+.
+.
+.TP
+\fB\-\-allow-replaced-filesystem\fI
+.
+By default, CryFS remembers file systems it has seen in this base directory and checks that it didn't get replaced by an attacker with an entirely different file system since the last time it was loaded. However, if you do want to replace the file system with an entirely new one, you can pass in this option to disable the check.
+.
+.
+.TP
+\fB\-\-missing-block-is-integrity-violation\fR=true
+.
+When CryFS encounters a missing ciphertext block, it cannot cannot (yet) know if it was deleted by an unauthorized adversary or by a second authorized client. This is one of the restrictions of the integrity checks currently in place. You can enable this flag to treat missing ciphertext blocks as integrity violations, but then your file system will not be usable by multiple clients anymore. By default, this flag is disabled.
+.
+.
+.TP
 \fB\-\-logfile\fR \fIfile\fR
 .
 Write status information to \fIfile\fR. If no logfile is given, CryFS will
@@ -205,6 +223,15 @@ By default, CryFS connects to the internet to check for known security
 vulnerabilities and new versions. This option disables this.
 .
 .
+.TP
+\fBCRYFS_LOCAL_STATE_DIR\fR=[path]
+.
+Sets the directory cryfs uses to store local state. This local state
+is used to recognize known file systems and run integrity checks
+(i.e. check that they haven't been modified by an attacker.
+Default value: ${HOME}/.cryfs
+.
+.
 .
 .SH SEE ALSO
 .

run-clang-tidy.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Note: Call this from a cmake build directory (e.g. cmake/) for out-of-source builds
+# Examples:
+# mkdir cmake && cd cmake && ../run-clang-tidy.sh
+# mkdir cmake && cd cmake && ../run-clang-tidy.sh -fix
+# mkdir cmake && cd cmake && ../run-clang-tidy.sh -export-fixes fixes.yaml
+
+set -e
+
+NUMCORES=`nproc`
+
+# Run cmake in current working directory, but on source that is in the same directory as this script file
+cmake -DBUILD_TESTING=on -DCMAKE_EXPORT_COMPILE_COMMANDS=ON "${0%/*}"
+
+# Build scrypt first. Our Makefiles call ino theirs, and this is needed to generate some header files. Clang-tidy will otherwise complain they're missing.
+make -j${NUMCORES} scrypt
+
+run-clang-tidy.py -j${NUMCORES} -quiet -header-filter "$(realpath ${0%/*})/(src|test)/.*" $@
+

src/CMakeLists.txt

@@ -8,3 +8,4 @@ add_subdirectory(blockstore)
 add_subdirectory(blobstore)
 add_subdirectory(cryfs)
 add_subdirectory(cryfs-cli)
+add_subdirectory(stats)

src/blobstore/CMakeLists.txt

@@ -11,6 +11,8 @@ set(SOURCES
     implementations/onblocks/datanodestore/DataInnerNode.cpp
     implementations/onblocks/datanodestore/DataNodeStore.cpp
     implementations/onblocks/datatreestore/impl/algorithms.cpp
+    implementations/onblocks/datatreestore/impl/LeafTraverser.cpp
+    implementations/onblocks/datatreestore/LeafHandle.cpp
     implementations/onblocks/datatreestore/DataTree.cpp
     implementations/onblocks/datatreestore/DataTreeStore.cpp
     implementations/onblocks/BlobOnBlocks.cpp

src/blobstore/implementations/onblocks/BlobOnBlocks.cpp

@@ -2,16 +2,19 @@
 #include "BlobOnBlocks.h"
 
 #include "datanodestore/DataLeafNode.h"
+#include "datanodestore/DataNodeStore.h"
 #include "utils/Math.h"
 #include <cmath>
 #include <cpp-utils/assert/assert.h>
+#include "datatreestore/LeafHandle.h"
 
 using std::function;
+using std::unique_lock;
+using std::mutex;
 using cpputils::unique_ref;
 using cpputils::Data;
-using blobstore::onblocks::datanodestore::DataLeafNode;
+using blockstore::BlockId;
-using blobstore::onblocks::datanodestore::DataNodeLayout;
+using blobstore::onblocks::datatreestore::LeafHandle;
-using blockstore::Key;
 
 namespace blobstore {
 namespace onblocks {
@@ -19,11 +22,11 @@ namespace onblocks {
 using parallelaccessdatatreestore::DataTreeRef;
 
 BlobOnBlocks::BlobOnBlocks(unique_ref<DataTreeRef> datatree)
-: _datatree(std::move(datatree)), _sizeCache(boost::none) {
+: _datatree(std::move(datatree)), _sizeCache(boost::none), _mutex() {
 }
 
 BlobOnBlocks::~BlobOnBlocks() {
-}
+} // NOLINT (workaround https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82481 )
 
 uint64_t BlobOnBlocks::size() const {
   if (_sizeCache == boost::none) {
@@ -37,22 +40,50 @@ void BlobOnBlocks::resize(uint64_t numBytes) {
   _sizeCache = numBytes;
 }
 
-void BlobOnBlocks::traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function<void (uint64_t, DataLeafNode *leaf, uint32_t, uint32_t)> func) const {
+void BlobOnBlocks::_traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function<void (uint64_t leafOffset, LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, function<Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const {
+  unique_lock<mutex> lock(_mutex); // TODO Multiple traverse calls in parallel?
   uint64_t endByte = beginByte + sizeBytes;
-  uint32_t firstLeaf = beginByte / _datatree->maxBytesPerLeaf();
+  uint64_t maxBytesPerLeaf = _datatree->maxBytesPerLeaf();
-  uint32_t endLeaf = utils::ceilDivision(endByte, _datatree->maxBytesPerLeaf());
+  uint32_t firstLeaf = beginByte / maxBytesPerLeaf;
-  bool writingOutside = size() < endByte; // TODO Calling size() is slow because it has to traverse the tree
+  uint32_t endLeaf = utils::ceilDivision(endByte, maxBytesPerLeaf);
-  _datatree->traverseLeaves(firstLeaf, endLeaf, [&func, beginByte, endByte, endLeaf, writingOutside](DataLeafNode *leaf, uint32_t leafIndex) {
+  bool blobIsGrowingFromThisTraversal = false;
-    uint64_t indexOfFirstLeafByte = leafIndex * leaf->maxStoreableBytes();
+  auto _onExistingLeaf = [&onExistingLeaf, beginByte, endByte, endLeaf, maxBytesPerLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex, bool isRightBorderLeaf, LeafHandle leafHandle) {
+      uint64_t indexOfFirstLeafByte = leafIndex * maxBytesPerLeaf;
+      ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
       uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
-    uint32_t dataEnd = std::min(leaf->maxStoreableBytes(), endByte - indexOfFirstLeafByte);
+      uint32_t dataEnd = std::min(maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
-    if (leafIndex == endLeaf-1 && writingOutside) {
+      // If we are traversing exactly until the last leaf, then the last leaf wasn't resized by the traversal and might have a wrong size. We have to fix it.
-      // If we are traversing an area that didn't exist before, then the last leaf was just created with a wrong size. We have to fix it.
+      if (isRightBorderLeaf) {
+        ASSERT(leafIndex == endLeaf-1, "If we traversed further right, this wouldn't be the right border leaf.");
+        auto leaf = leafHandle.node();
+        if (leaf->numBytes() < dataEnd) {
           leaf->resize(dataEnd);
+          blobIsGrowingFromThisTraversal = true;
         }
-    func(indexOfFirstLeafByte, leaf, dataBegin, dataEnd-dataBegin);
+      }
-  });
+      onExistingLeaf(indexOfFirstLeafByte, std::move(leafHandle), dataBegin, dataEnd-dataBegin);
-  if (writingOutside) {
+  };
+  auto _onCreateLeaf = [&onCreateLeaf, maxBytesPerLeaf, beginByte, firstLeaf, endByte, endLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex) -> Data {
+      blobIsGrowingFromThisTraversal = true;
+      uint64_t indexOfFirstLeafByte = leafIndex * maxBytesPerLeaf;
+      ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
+      uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
+      uint32_t dataEnd = std::min(maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
+      ASSERT(leafIndex == firstLeaf || dataBegin == 0, "Only the leftmost leaf can have a gap on the left.");
+      ASSERT(leafIndex == endLeaf-1 || dataEnd == maxBytesPerLeaf, "Only the rightmost leaf can have a gap on the right");
+      Data data = onCreateLeaf(indexOfFirstLeafByte + dataBegin, dataEnd-dataBegin);
+      ASSERT(data.size() == dataEnd-dataBegin, "Returned leaf data with wrong size");
+      // If this leaf is created but only partly in the traversed region (i.e. dataBegin > leafBegin), we have to fill the data before the traversed region with zeroes.
+      if (dataBegin != 0) {
+        Data actualData(dataBegin + data.size());
+        std::memset(actualData.data(), 0, dataBegin);
+        std::memcpy(actualData.dataOffset(dataBegin), data.data(), data.size());
+        data = std::move(actualData);
+      }
+      return data;
+  };
+  _datatree->traverseLeaves(firstLeaf, endLeaf, _onExistingLeaf, _onCreateLeaf);
+  if (blobIsGrowingFromThisTraversal) {
     ASSERT(_datatree->numStoredBytes() == endByte, "Writing didn't grow by the correct number of bytes");
     _sizeCache = endByte;
   }
@@ -67,38 +98,60 @@ Data BlobOnBlocks::readAll() const {
 }
 
 void BlobOnBlocks::read(void *target, uint64_t offset, uint64_t count) const {
-  ASSERT(offset <= size() && offset + count <= size(), "BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
+  uint64_t _size = size();
+  ASSERT(offset <= _size && offset + count <= _size, "BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
   uint64_t read = tryRead(target, offset, count);
   ASSERT(read == count, "BlobOnBlocks::read() couldn't read all requested bytes. Use BlobOnBlocks::tryRead() if this should be allowed.");
 }
 
 uint64_t BlobOnBlocks::tryRead(void *target, uint64_t offset, uint64_t count) const {
   //TODO Quite inefficient to call size() here, because that has to traverse the tree
-  uint64_t realCount = std::max(UINT64_C(0), std::min(count, size()-offset));
+  uint64_t realCount = std::max(INT64_C(0), std::min(static_cast<int64_t>(count), static_cast<int64_t>(size())-static_cast<int64_t>(offset)));
   _read(target, offset, realCount);
   return realCount;
 }
 
 void BlobOnBlocks::_read(void *target, uint64_t offset, uint64_t count) const {
-  traverseLeaves(offset, count, [target, offset] (uint64_t indexOfFirstLeafByte, const DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
+  auto onExistingLeaf = [target, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
+      ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Writing to target out of bounds");
       //TODO Simplify formula, make it easier to understand
-      leaf->read((uint8_t*)target + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
+      leaf.node()->read(static_cast<uint8_t*>(target) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
-  });
+  };
+  auto onCreateLeaf = [] (uint64_t /*beginByte*/, uint32_t /*count*/) -> Data {
+      ASSERT(false, "Reading shouldn't create new leaves.");
+  };
+  _traverseLeaves(offset, count, onExistingLeaf, onCreateLeaf);
 }
 
 void BlobOnBlocks::write(const void *source, uint64_t offset, uint64_t count) {
-  traverseLeaves(offset, count, [source, offset] (uint64_t indexOfFirstLeafByte, DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
+  auto onExistingLeaf = [source, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
+      ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Reading from source out of bounds");
+      if (leafDataOffset == 0 && leafDataSize == leaf.nodeStore()->layout().maxBytesPerLeaf()) {
+        Data leafData(leafDataSize);
+        std::memcpy(leafData.data(), static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset, leafDataSize);
+        leaf.nodeStore()->overwriteLeaf(leaf.blockId(), std::move(leafData));
+      } else {
             //TODO Simplify formula, make it easier to understand
-    leaf->write((uint8_t*)source + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
+        leaf.node()->write(static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset,
-  });
+                           leafDataSize);
+      }
+  };
+  auto onCreateLeaf = [source, offset, count] (uint64_t beginByte, uint32_t numBytes) -> Data {
+      ASSERT(beginByte >= offset && beginByte-offset <= count && beginByte-offset+numBytes <= count, "Reading from source out of bounds");
+      Data result(numBytes);
+      //TODO Simplify formula, make it easier to understand
+      std::memcpy(result.data(), static_cast<const uint8_t*>(source) + beginByte - offset, numBytes);
+      return result;
+  };
+  _traverseLeaves(offset, count, onExistingLeaf, onCreateLeaf);
 }
 
 void BlobOnBlocks::flush() {
   _datatree->flush();
 }
 
-const Key &BlobOnBlocks::key() const {
+const BlockId &BlobOnBlocks::blockId() const {
-  return _datatree->key();
+  return _datatree->blockId();
 }
 
 unique_ref<DataTreeRef> BlobOnBlocks::releaseTree() {

src/blobstore/implementations/onblocks/BlobOnBlocks.h

@@ -3,6 +3,7 @@
 #define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_BLOBONBLOCKS_H_
 
 #include "../../interface/Blob.h"
+#include "datatreestore/LeafHandle.h"
 
 #include <memory>
 #include <boost/optional.hpp>
@@ -21,7 +22,7 @@ public:
   BlobOnBlocks(cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> datatree);
   ~BlobOnBlocks();
 
-  const blockstore::Key &key() const override;
+  const blockstore::BlockId &blockId() const override;
 
   uint64_t size() const override;
   void resize(uint64_t numBytes) override;
@@ -38,10 +39,11 @@ public:
 private:
 
   void _read(void *target, uint64_t offset, uint64_t count) const;
-  void traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function<void (uint64_t, datanodestore::DataLeafNode *, uint32_t, uint32_t)>) const;
+  void _traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function<void (uint64_t leafOffset, datatreestore::LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, std::function<cpputils::Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const;
 
   cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> _datatree;
   mutable boost::optional<uint64_t> _sizeCache;
+  mutable std::mutex _mutex;
 
   DISALLOW_COPY_AND_ASSIGN(BlobOnBlocks);
 };

src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp

@@ -15,7 +15,7 @@ using cpputils::make_unique_ref;
 
 using blockstore::BlockStore;
 using blockstore::parallelaccess::ParallelAccessBlockStore;
-using blockstore::Key;
+using blockstore::BlockId;
 using cpputils::dynamic_pointer_move;
 using boost::optional;
 using boost::none;
@@ -38,8 +38,8 @@ unique_ref<Blob> BlobStoreOnBlocks::create() {
     return make_unique_ref<BlobOnBlocks>(_dataTreeStore->createNewTree());
 }
 
-optional<unique_ref<Blob>> BlobStoreOnBlocks::load(const Key &key) {
+optional<unique_ref<Blob>> BlobStoreOnBlocks::load(const BlockId &blockId) {
-    auto tree = _dataTreeStore->load(key);
+    auto tree = _dataTreeStore->load(blockId);
     if (tree == none) {
         return none;
     }
@@ -52,6 +52,10 @@ void BlobStoreOnBlocks::remove(unique_ref<Blob> blob) {
     _dataTreeStore->remove((*_blob)->releaseTree());
 }
 
+void BlobStoreOnBlocks::remove(const BlockId &blockId) {
+    _dataTreeStore->remove(blockId);
+}
+
 uint64_t BlobStoreOnBlocks::virtualBlocksizeBytes() const {
     return _dataTreeStore->virtualBlocksizeBytes();
 }

src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h

@@ -20,9 +20,10 @@ public:
   ~BlobStoreOnBlocks();
 
   cpputils::unique_ref<Blob> create() override;
-  boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::Key &key) override;
+  boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::BlockId &blockId) override;
 
   void remove(cpputils::unique_ref<Blob> blob) override;
+  void remove(const blockstore::BlockId &blockId) override;
 
   //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
   //virtual means "space we can use" as opposed to "space it takes on the disk" (i.e. virtual is without headers, checksums, ...)

src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp

@@ -3,10 +3,12 @@
 #include <cpp-utils/assert/assert.h>
 
 using blockstore::Block;
+using blockstore::BlockStore;
 using cpputils::Data;
 using cpputils::unique_ref;
 using cpputils::make_unique_ref;
-using blockstore::Key;
+using blockstore::BlockId;
+using std::vector;
 
 namespace blobstore {
 namespace onblocks {
@@ -23,62 +25,62 @@ DataInnerNode::DataInnerNode(DataNodeView view)
 DataInnerNode::~DataInnerNode() {
 }
 
-unique_ref<DataInnerNode> DataInnerNode::InitializeNewNode(unique_ref<Block> block, const DataNode &first_child) {
+unique_ref<DataInnerNode> DataInnerNode::InitializeNewNode(unique_ref<Block> block, const DataNodeLayout &layout, uint8_t depth, const vector<BlockId> &children) {
-  DataNodeView node(std::move(block));
+  ASSERT(children.size() >= 1, "An inner node must have at least one child");
-  node.setFormatVersion(DataNode::FORMAT_VERSION_HEADER);
+  Data data = _serializeChildren(children);
-  node.setDepth(first_child.depth() + 1);
+
-  node.setSize(1);
+  return make_unique_ref<DataInnerNode>(DataNodeView::initialize(std::move(block), layout, DataNode::FORMAT_VERSION_HEADER, depth, children.size(), std::move(data)));
-  auto result = make_unique_ref<DataInnerNode>(std::move(node));
+}
-  result->ChildrenBegin()->setKey(first_child.key());
+
-  return result;
+unique_ref<DataInnerNode> DataInnerNode::CreateNewNode(BlockStore *blockStore, const DataNodeLayout &layout, uint8_t depth, const vector<BlockId> &children) {
+  ASSERT(children.size() >= 1, "An inner node must have at least one child");
+  Data data = _serializeChildren(children);
+
+  return make_unique_ref<DataInnerNode>(DataNodeView::create(blockStore, layout, DataNode::FORMAT_VERSION_HEADER, depth, children.size(), std::move(data)));
+}
+
+Data DataInnerNode::_serializeChildren(const vector<BlockId> &children) {
+  Data data(sizeof(ChildEntry) * children.size());
+  uint32_t i = 0;
+  for (const BlockId &child : children) {
+    child.ToBinary(data.dataOffset(i * BlockId::BINARY_LENGTH));
+    ++i;
+  }
+  return data;
 }
 
 uint32_t DataInnerNode::numChildren() const {
   return node().Size();
 }
 
-DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() {
+DataInnerNode::ChildEntry DataInnerNode::readChild(unsigned int index) const {
-  return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->ChildrenBegin());
-}
-
-const DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() const {
-  return node().DataBegin<ChildEntry>();
-}
-
-DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() {
-  return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->ChildrenEnd());
-}
-
-const DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() const {
-  return ChildrenBegin() + node().Size();
-}
-
-DataInnerNode::ChildEntry *DataInnerNode::LastChild() {
-  return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->LastChild());
-}
-
-const DataInnerNode::ChildEntry *DataInnerNode::LastChild() const {
-  return getChild(numChildren()-1);
-}
-
-DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) {
-  return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->getChild(index));
-}
-
-const DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) const {
   ASSERT(index < numChildren(), "Accessing child out of range");
-  return ChildrenBegin()+index;
+  return ChildEntry(BlockId::FromBinary(static_cast<const uint8_t*>(node().data()) + index * sizeof(ChildEntry)));
+}
+
+void DataInnerNode::_writeChild(unsigned int index, const ChildEntry& child) {
+  ASSERT(index < numChildren(), "Accessing child out of range");
+  node().write(child.blockId().data().data(), index * sizeof(ChildEntry), sizeof(ChildEntry));
+}
+
+DataInnerNode::ChildEntry DataInnerNode::readLastChild() const {
+  return readChild(numChildren() - 1);
+}
+
+void DataInnerNode::_writeLastChild(const ChildEntry& child) {
+  _writeChild(numChildren() - 1, child);
 }
 
 void DataInnerNode::addChild(const DataNode &child) {
   ASSERT(numChildren() < maxStoreableChildren(), "Adding more children than we can store");
   ASSERT(child.depth() == depth()-1, "The child that should be added has wrong depth");
   node().setSize(node().Size()+1);
-  LastChild()->setKey(child.key());
+  _writeLastChild(ChildEntry(child.blockId()));
 }
 
 void DataInnerNode::removeLastChild() {
   ASSERT(node().Size() > 1, "There is no child to remove");
+  _writeLastChild(ChildEntry(BlockId::Null()));
   node().setSize(node().Size()-1);
 }
 

src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.h

@@ -11,33 +11,29 @@ namespace datanodestore {
 
 class DataInnerNode final: public DataNode {
 public:
-  static cpputils::unique_ref<DataInnerNode> InitializeNewNode(cpputils::unique_ref<blockstore::Block> block, const DataNode &first_child_key);
+  static cpputils::unique_ref<DataInnerNode> InitializeNewNode(cpputils::unique_ref<blockstore::Block> block, const DataNodeLayout &layout, uint8_t depth, const std::vector<blockstore::BlockId> &children);
+  static cpputils::unique_ref<DataInnerNode> CreateNewNode(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, uint8_t depth, const std::vector<blockstore::BlockId> &children);
+
+  using ChildEntry = DataInnerNode_ChildEntry;
 
   DataInnerNode(DataNodeView block);
   ~DataInnerNode();
 
-  using ChildEntry = DataInnerNode_ChildEntry;
-
   uint32_t maxStoreableChildren() const;
 
-  ChildEntry *getChild(unsigned int index);
+  ChildEntry readChild(unsigned int index) const;
-  const ChildEntry *getChild(unsigned int index) const;
+  ChildEntry readLastChild() const;
 
   uint32_t numChildren() const;
 
-  void addChild(const DataNode &child_key);
+  void addChild(const DataNode &child_blockId);
 
   void removeLastChild();
 
-  ChildEntry *LastChild();
-  const ChildEntry *LastChild() const;
-
 private:
-
+  void _writeChild(unsigned int index, const ChildEntry& child);
-  ChildEntry *ChildrenBegin();
+  void _writeLastChild(const ChildEntry& child);
-  ChildEntry *ChildrenEnd();
+  static cpputils::Data _serializeChildren(const std::vector<blockstore::BlockId> &children);
-  const ChildEntry *ChildrenBegin() const;
-  const ChildEntry *ChildrenEnd() const;
 
   DISALLOW_COPY_AND_ASSIGN(DataInnerNode);
 };

src/blobstore/implementations/onblocks/datanodestore/DataInnerNode_ChildEntry.h

@@ -10,17 +10,19 @@ namespace datanodestore{
 
 struct DataInnerNode_ChildEntry final {
 public:
-  blockstore::Key key() const {
+  DataInnerNode_ChildEntry(const blockstore::BlockId &blockId): _blockId(blockId) {}
-    return blockstore::Key::FromBinary(_keydata);
-  }
-private:
-  void setKey(const blockstore::Key &key) {
-    key.ToBinary(_keydata);
-  }
-  friend class DataInnerNode;
-  uint8_t _keydata[blockstore::Key::BINARY_LENGTH];
 
-  DISALLOW_COPY_AND_ASSIGN(DataInnerNode_ChildEntry);
+  const blockstore::BlockId& blockId() const {
+    return _blockId;
+  }
+
+  DataInnerNode_ChildEntry(const DataInnerNode_ChildEntry&) = delete;
+  DataInnerNode_ChildEntry& operator=(const DataInnerNode_ChildEntry&) = delete;
+  DataInnerNode_ChildEntry(DataInnerNode_ChildEntry&&) = default;
+  DataInnerNode_ChildEntry& operator=(DataInnerNode_ChildEntry&&) = default;
+
+private:
+  blockstore::BlockId _blockId;
 };
 
 }

src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.cpp

@@ -2,9 +2,9 @@
 #include "DataInnerNode.h"
 #include <cpp-utils/assert/assert.h>
 
-using blockstore::Block;
 using cpputils::Data;
-using blockstore::Key;
+using blockstore::BlockId;
+using blockstore::BlockStore;
 using cpputils::unique_ref;
 using cpputils::make_unique_ref;
 
@@ -24,18 +24,21 @@ DataLeafNode::DataLeafNode(DataNodeView view)
 DataLeafNode::~DataLeafNode() {
 }
 
-unique_ref<DataLeafNode> DataLeafNode::InitializeNewNode(unique_ref<Block> block) {
+unique_ref<DataLeafNode> DataLeafNode::CreateNewNode(BlockStore *blockStore, const DataNodeLayout &layout, Data data) {
-  DataNodeView node(std::move(block));
+  ASSERT(data.size() <= layout.maxBytesPerLeaf(), "Data passed in is too large for one leaf.");
-  node.setFormatVersion(DataNode::FORMAT_VERSION_HEADER);
+  uint32_t size = data.size();
-  node.setDepth(0);
+  return make_unique_ref<DataLeafNode>(DataNodeView::create(blockStore, layout, DataNode::FORMAT_VERSION_HEADER, 0, size, std::move(data)));
-  node.setSize(0);
+}
-  //fillDataWithZeroes(); not needed, because a newly created block will be zeroed out. DataLeafNodeTest.SpaceIsZeroFilledWhenGrowing ensures this.
+
-  return make_unique_ref<DataLeafNode>(std::move(node));
+unique_ref<DataLeafNode> DataLeafNode::OverwriteNode(BlockStore *blockStore, const DataNodeLayout &layout, const BlockId &blockId, Data data) {
+  ASSERT(data.size() == layout.maxBytesPerLeaf(), "Data passed in is too large for one leaf.");
+  uint32_t size = data.size();
+  return make_unique_ref<DataLeafNode>(DataNodeView::overwrite(blockStore, layout, DataNode::FORMAT_VERSION_HEADER, 0, size, blockId, std::move(data)));
 }
 
 void DataLeafNode::read(void *target, uint64_t offset, uint64_t size) const {
   ASSERT(offset <= node().Size() && offset + size <= node().Size(), "Read out of valid area"); // Also check offset, because the addition could lead to overflows
-  std::memcpy(target, (uint8_t*)node().data() + offset, size);
+  std::memcpy(target, static_cast<const uint8_t*>(node().data()) + offset, size);
 }
 
 void DataLeafNode::write(const void *source, uint64_t offset, uint64_t size) {

src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.h

@@ -11,7 +11,8 @@ class DataInnerNode;
 
 class DataLeafNode final: public DataNode {
 public:
-  static cpputils::unique_ref<DataLeafNode> InitializeNewNode(cpputils::unique_ref<blockstore::Block> block);
+  static cpputils::unique_ref<DataLeafNode> CreateNewNode(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, cpputils::Data data);
+  static cpputils::unique_ref<DataLeafNode> OverwriteNode(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, const blockstore::BlockId &blockId, cpputils::Data data);
 
   DataLeafNode(DataNodeView block);
   ~DataLeafNode();

src/blobstore/implementations/onblocks/datanodestore/DataNode.cpp

@@ -4,10 +4,8 @@
 #include "DataNodeStore.h"
 #include <blockstore/utils/BlockStoreUtils.h>
 
-using blockstore::Block;
+using blockstore::BlockId;
-using blockstore::Key;
 
-using std::runtime_error;
 using cpputils::unique_ref;
 
 namespace blobstore {
@@ -31,20 +29,19 @@ const DataNodeView &DataNode::node() const {
   return _node;
 }
 
-const Key &DataNode::key() const {
+const BlockId &DataNode::blockId() const {
-  return _node.key();
+  return _node.blockId();
 }
 
 uint8_t DataNode::depth() const {
   return _node.Depth();
 }
 
-unique_ref<DataInnerNode> DataNode::convertToNewInnerNode(unique_ref<DataNode> node, const DataNode &first_child) {
+unique_ref<DataInnerNode> DataNode::convertToNewInnerNode(unique_ref<DataNode> node, const DataNodeLayout &layout, const DataNode &first_child) {
-  Key key = node->key();
   auto block = node->_node.releaseBlock();
   blockstore::utils::fillWithZeroes(block.get());
 
-  return DataInnerNode::InitializeNewNode(std::move(block), first_child);
+  return DataInnerNode::InitializeNewNode(std::move(block), layout, first_child.depth()+1, {first_child.blockId()});
 }
 
 void DataNode::flush() const {

src/blobstore/implementations/onblocks/datanodestore/DataNode.h

@@ -15,11 +15,11 @@ class DataNode {
 public:
   virtual ~DataNode();
 
-  const blockstore::Key &key() const;
+  const blockstore::BlockId &blockId() const;
 
   uint8_t depth() const;
 
-  static cpputils::unique_ref<DataInnerNode> convertToNewInnerNode(cpputils::unique_ref<DataNode> node, const DataNode &first_child);
+  static cpputils::unique_ref<DataInnerNode> convertToNewInnerNode(cpputils::unique_ref<DataNode> node, const DataNodeLayout &layout, const DataNode &first_child);
 
   void flush() const;
 

src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp

@@ -8,13 +8,15 @@
 
 using blockstore::BlockStore;
 using blockstore::Block;
-using blockstore::Key;
+using blockstore::BlockId;
 using cpputils::Data;
 using cpputils::unique_ref;
 using cpputils::make_unique_ref;
+using cpputils::dynamic_pointer_move;
 using std::runtime_error;
 using boost::optional;
 using boost::none;
+using std::vector;
 
 namespace blobstore {
 namespace onblocks {
@@ -28,7 +30,6 @@ DataNodeStore::~DataNodeStore() {
 }
 
 unique_ref<DataNode> DataNodeStore::load(unique_ref<Block> block) {
-  ASSERT(block->size() == _layout.blocksizeBytes(), "Loading block of wrong size");
   DataNodeView node(std::move(block));
 
   if (node.Depth() == 0) {
@@ -40,24 +41,25 @@ unique_ref<DataNode> DataNodeStore::load(unique_ref<Block> block) {
   }
 }
 
-unique_ref<DataInnerNode> DataNodeStore::createNewInnerNode(const DataNode &first_child) {
+unique_ref<DataInnerNode> DataNodeStore::createNewInnerNode(uint8_t depth, const vector<BlockId> &children) {
-  ASSERT(first_child.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
+  ASSERT(children.size() >= 1, "Inner node must have at least one child");
-  //TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
+  return DataInnerNode::CreateNewNode(_blockstore.get(), _layout, depth, children);
-  auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
-  return DataInnerNode::InitializeNewNode(std::move(block), first_child);
 }
 
-unique_ref<DataLeafNode> DataNodeStore::createNewLeafNode() {
+unique_ref<DataLeafNode> DataNodeStore::createNewLeafNode(Data data) {
-  //TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
+  return DataLeafNode::CreateNewNode(_blockstore.get(), _layout, std::move(data));
-  auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
-  return DataLeafNode::InitializeNewNode(std::move(block));
 }
 
-optional<unique_ref<DataNode>> DataNodeStore::load(const Key &key) {
+unique_ref<DataLeafNode> DataNodeStore::overwriteLeaf(const BlockId &blockId, Data data) {
-  auto block = _blockstore->load(key);
+  return DataLeafNode::OverwriteNode(_blockstore.get(), _layout, blockId, std::move(data));
+}
+
+optional<unique_ref<DataNode>> DataNodeStore::load(const BlockId &blockId) {
+  auto block = _blockstore->load(blockId);
   if (block == none) {
     return none;
   } else {
+    ASSERT((*block)->size() == _layout.blocksizeBytes(), "Loading block of wrong size");
     return load(std::move(*block));
   }
 }
@@ -71,21 +73,52 @@ unique_ref<DataNode> DataNodeStore::createNewNodeAsCopyFrom(const DataNode &sour
 unique_ref<DataNode> DataNodeStore::overwriteNodeWith(unique_ref<DataNode> target, const DataNode &source) {
   ASSERT(target->node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Target node has wrong layout. Is it from the same DataNodeStore?");
   ASSERT(source.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
-  Key key = target->key();
-  {
   auto targetBlock = target->node().releaseBlock();
   cpputils::destruct(std::move(target)); // Call destructor
   blockstore::utils::copyTo(targetBlock.get(), source.node().block());
-  }
+  return DataNodeStore::load(std::move(targetBlock));
-  auto loaded = load(key);
-  ASSERT(loaded != none, "Couldn't load the target node after overwriting it");
-  return std::move(*loaded);
 }
 
 void DataNodeStore::remove(unique_ref<DataNode> node) {
-  auto block = node->node().releaseBlock();
+  BlockId blockId = node->blockId();
-  cpputils::destruct(std::move(node)); // Call destructor
+  cpputils::destruct(std::move(node));
-  _blockstore->remove(std::move(block));
+  remove(blockId);
+}
+
+void DataNodeStore::remove(const BlockId &blockId) {
+  _blockstore->remove(blockId);
+}
+
+void DataNodeStore::removeSubtree(unique_ref<DataNode> node) {
+  auto leaf = dynamic_pointer_move<DataLeafNode>(node);
+  if (leaf != none) {
+    remove(std::move(*leaf));
+    return;
+  }
+
+  auto inner = dynamic_pointer_move<DataInnerNode>(node);
+  ASSERT(inner != none, "Is neither a leaf nor an inner node");
+  for (uint32_t i = 0; i < (*inner)->numChildren(); ++i) {
+    removeSubtree((*inner)->depth()-1, (*inner)->readChild(i).blockId());
+  }
+  remove(std::move(*inner));
+}
+
+void DataNodeStore::removeSubtree(uint8_t depth, const BlockId &blockId) {
+  if (depth == 0) {
+    remove(blockId);
+  } else {
+    auto node = load(blockId);
+    ASSERT(node != none, "Node for removeSubtree not found");
+
+    auto inner = dynamic_pointer_move<DataInnerNode>(*node);
+    ASSERT(inner != none, "Is not an inner node, but depth was not zero");
+    ASSERT((*inner)->depth() == depth, "Wrong depth given");
+    for (uint32_t i = 0; i < (*inner)->numChildren(); ++i) {
+      removeSubtree(depth-1, (*inner)->readChild(i).blockId());
+    }
+    remove(std::move(*inner));
+  }
 }
 
 uint64_t DataNodeStore::numNodes() const {
@@ -100,19 +133,6 @@ uint64_t DataNodeStore::virtualBlocksizeBytes() const {
   return _layout.blocksizeBytes();
 }
 
-void DataNodeStore::removeSubtree(unique_ref<DataNode> node) {
-  //TODO Make this faster by not loading the leaves but just deleting them. Can be recognized, because of the depth of their parents.
-  DataInnerNode *inner = dynamic_cast<DataInnerNode*>(node.get());
-  if (inner != nullptr) {
-    for (uint32_t i = 0; i < inner->numChildren(); ++i) {
-      auto child = load(inner->getChild(i)->key());
-      ASSERT(child != none, "Couldn't load child node");
-      removeSubtree(std::move(*child));
-    }
-  }
-  remove(std::move(node));
-}
-
 DataNodeLayout DataNodeStore::layout() const {
   return _layout;
 }

src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h

@@ -5,7 +5,7 @@
 #include <memory>
 #include <cpp-utils/macros.h>
 #include "DataNodeView.h"
-#include <blockstore/utils/Key.h>
+#include <blockstore/utils/BlockId.h>
 
 namespace blockstore{
 class Block;
@@ -28,17 +28,21 @@ public:
 
   DataNodeLayout layout() const;
 
-  boost::optional<cpputils::unique_ref<DataNode>> load(const blockstore::Key &key);
+  boost::optional<cpputils::unique_ref<DataNode>> load(const blockstore::BlockId &blockId);
+  static cpputils::unique_ref<DataNode> load(cpputils::unique_ref<blockstore::Block> block);
 
-  cpputils::unique_ref<DataLeafNode> createNewLeafNode();
+  cpputils::unique_ref<DataLeafNode> createNewLeafNode(cpputils::Data data);
-  cpputils::unique_ref<DataInnerNode> createNewInnerNode(const DataNode &first_child);
+  cpputils::unique_ref<DataInnerNode> createNewInnerNode(uint8_t depth, const std::vector<blockstore::BlockId> &children);
 
   cpputils::unique_ref<DataNode> createNewNodeAsCopyFrom(const DataNode &source);
 
   cpputils::unique_ref<DataNode> overwriteNodeWith(cpputils::unique_ref<DataNode> target, const DataNode &source);
 
-  void remove(cpputils::unique_ref<DataNode> node);
+  cpputils::unique_ref<DataLeafNode> overwriteLeaf(const blockstore::BlockId &blockId, cpputils::Data data);
 
+  void remove(cpputils::unique_ref<DataNode> node);
+  void remove(const blockstore::BlockId &blockId);
+  void removeSubtree(uint8_t depth, const blockstore::BlockId &blockId);
   void removeSubtree(cpputils::unique_ref<DataNode> node);
 
   //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
@@ -48,7 +52,6 @@ public:
   //TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree()
 
 private:
-  cpputils::unique_ref<DataNode> load(cpputils::unique_ref<blockstore::Block> block);
 
   cpputils::unique_ref<blockstore::BlockStore> _blockstore;
   const DataNodeLayout _layout;

src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h

@@ -65,10 +65,33 @@ public:
   }
   ~DataNodeView() {}
 
+  static DataNodeView create(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, uint16_t formatVersion, uint8_t depth, uint32_t size, cpputils::Data data) {
+    ASSERT(data.size() <= layout.datasizeBytes(), "Data is too large for node");
+    cpputils::Data serialized = _serialize(layout, formatVersion, depth, size, std::move(data));
+    ASSERT(serialized.size() == layout.blocksizeBytes(), "Wrong block size");
+    auto block = blockStore->create(serialized);
+    return DataNodeView(std::move(block));
+  }
+
+  static DataNodeView initialize(cpputils::unique_ref<blockstore::Block> block, const DataNodeLayout &layout, uint16_t formatVersion, uint8_t depth, uint32_t size, cpputils::Data data) {
+    ASSERT(data.size() <= DataNodeLayout(block->size()).datasizeBytes(), "Data is too large for node");
+    cpputils::Data serialized = _serialize(layout, formatVersion, depth, size, std::move(data));
+    ASSERT(serialized.size() == block->size(), "Block has wrong size");
+    block->write(serialized.data(), 0, serialized.size());
+    return DataNodeView(std::move(block));
+  }
+
+  static DataNodeView overwrite(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, uint16_t formatVersion, uint8_t depth, uint32_t size, const blockstore::BlockId &blockId, cpputils::Data data) {
+    ASSERT(data.size() <= layout.datasizeBytes(), "Data is too large for node");
+    cpputils::Data serialized = _serialize(layout, formatVersion, depth, size, std::move(data));
+    auto block = blockStore->overwrite(blockId, std::move(serialized));
+    return DataNodeView(std::move(block));
+  }
+
   DataNodeView(DataNodeView &&rhs) = default;
 
   uint16_t FormatVersion() const {
-    return *((uint8_t*)_block->data()+DataNodeLayout::FORMAT_VERSION_OFFSET_BYTES);
+    return cpputils::deserializeWithOffset<uint16_t>(_block->data(), DataNodeLayout::FORMAT_VERSION_OFFSET_BYTES);
   }
 
   void setFormatVersion(uint16_t value) {
@@ -76,7 +99,7 @@ public:
   }
 
   uint8_t Depth() const {
-    return *((uint8_t*)_block->data()+DataNodeLayout::DEPTH_OFFSET_BYTES);
+    return cpputils::deserializeWithOffset<uint8_t>(_block->data(), DataNodeLayout::DEPTH_OFFSET_BYTES);
   }
 
   void setDepth(uint8_t value) {
@@ -84,7 +107,7 @@ public:
   }
 
   uint32_t Size() const {
-    return *(uint32_t*)((uint8_t*)_block->data()+DataNodeLayout::SIZE_OFFSET_BYTES);
+    return cpputils::deserializeWithOffset<uint32_t>(_block->data(), DataNodeLayout::SIZE_OFFSET_BYTES);
   }
 
   void setSize(uint32_t value) {
@@ -92,24 +115,13 @@ public:
   }
 
   const void *data() const {
-    return (uint8_t*)_block->data() + DataNodeLayout::HEADERSIZE_BYTES;
+    return static_cast<const uint8_t*>(_block->data()) + DataNodeLayout::HEADERSIZE_BYTES;
   }
 
   void write(const void *source, uint64_t offset, uint64_t size) {
     _block->write(source, offset + DataNodeLayout::HEADERSIZE_BYTES, size);
   }
 
-  template<typename Entry>
-  const Entry *DataBegin() const {
-    return GetOffset<DataNodeLayout::HEADERSIZE_BYTES, Entry>();
-  }
-
-  template<typename Entry>
-  const Entry *DataEnd() const {
-    const unsigned int NUM_ENTRIES = layout().datasizeBytes() / sizeof(Entry);
-    return DataBegin<Entry>() + NUM_ENTRIES;
-  }
-
   DataNodeLayout layout() const {
     return DataNodeLayout(_block->size());
   }
@@ -122,8 +134,8 @@ public:
     return *_block;
   }
 
-  const blockstore::Key &key() const {
+  const blockstore::BlockId &blockId() const {
-    return _block->key();
+    return _block->blockId();
   }
 
   void flush() const {
@@ -131,9 +143,14 @@ public:
   }
 
 private:
-  template<int offset, class Type>
+  static cpputils::Data _serialize(const DataNodeLayout &layout, uint16_t formatVersion, uint8_t depth, uint32_t size, cpputils::Data data) {
-  const Type *GetOffset() const {
+    cpputils::Data result(layout.blocksizeBytes());
-    return (Type*)(((const int8_t*)_block->data())+offset);
+    cpputils::serialize<uint16_t>(result.dataOffset(layout.FORMAT_VERSION_OFFSET_BYTES), formatVersion);
+    cpputils::serialize<uint8_t>(result.dataOffset(layout.DEPTH_OFFSET_BYTES), depth);
+    cpputils::serialize<uint32_t>(result.dataOffset(layout.SIZE_OFFSET_BYTES), size);
+    std::memcpy(result.dataOffset(layout.HEADERSIZE_BYTES), data.data(), data.size());
+    std::memset(result.dataOffset(layout.HEADERSIZE_BYTES+data.size()), 0, layout.datasizeBytes()-data.size());
+    return result;
   }
 
   cpputils::unique_ref<blockstore::Block> _block;

src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp

@@ -11,118 +11,37 @@
 #include <cpp-utils/pointer/optional_ownership_ptr.h>
 #include <cmath>
 #include <cpp-utils/assert/assert.h>
+#include "impl/LeafTraverser.h"
 
-using blockstore::Key;
+using blockstore::BlockId;
 using blobstore::onblocks::datanodestore::DataNodeStore;
 using blobstore::onblocks::datanodestore::DataNode;
 using blobstore::onblocks::datanodestore::DataInnerNode;
 using blobstore::onblocks::datanodestore::DataLeafNode;
-using blobstore::onblocks::datanodestore::DataNodeLayout;
 
-using std::dynamic_pointer_cast;
 using std::function;
 using boost::shared_mutex;
 using boost::shared_lock;
 using boost::unique_lock;
 using boost::none;
-using std::vector;
 
-using cpputils::dynamic_pointer_move;
 using cpputils::optional_ownership_ptr;
-using cpputils::WithOwnership;
-using cpputils::WithoutOwnership;
 using cpputils::unique_ref;
+using cpputils::Data;
 
 namespace blobstore {
 namespace onblocks {
 namespace datatreestore {
 
 DataTree::DataTree(DataNodeStore *nodeStore, unique_ref<DataNode> rootNode)
-  : _mutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)) {
+  : _mutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)), _blockId(_rootNode->blockId()), _numLeavesCache(none) {
 }
 
 DataTree::~DataTree() {
 }
 
-void DataTree::removeLastDataLeaf() {
+const BlockId &DataTree::blockId() const {
-  auto deletePosOrNull = algorithms::GetLowestRightBorderNodeWithMoreThanOneChildOrNull(_nodeStore, _rootNode.get());
+  return _blockId;
-  ASSERT(deletePosOrNull.get() != nullptr, "Tree has only one leaf, can't shrink it.");
-
-  deleteLastChildSubtree(deletePosOrNull.get());
-
-  ifRootHasOnlyOneChildReplaceRootWithItsChild();
-}
-
-void DataTree::ifRootHasOnlyOneChildReplaceRootWithItsChild() {
-  DataInnerNode *rootNode = dynamic_cast<DataInnerNode*>(_rootNode.get());
-  ASSERT(rootNode != nullptr, "RootNode is not an inner node");
-  if (rootNode->numChildren() == 1) {
-    auto child = _nodeStore->load(rootNode->getChild(0)->key());
-    ASSERT(child != none, "Couldn't load first child of root node");
-    _rootNode = _nodeStore->overwriteNodeWith(std::move(_rootNode), **child);
-    _nodeStore->remove(std::move(*child));
-  }
-}
-
-void DataTree::deleteLastChildSubtree(DataInnerNode *node) {
-  auto lastChild = _nodeStore->load(node->LastChild()->key());
-  ASSERT(lastChild != none, "Couldn't load last child");
-  _nodeStore->removeSubtree(std::move(*lastChild));
-  node->removeLastChild();
-}
-
-unique_ref<DataLeafNode> DataTree::addDataLeaf() {
-  auto insertPosOrNull = algorithms::GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(_nodeStore, _rootNode.get());
-  if (insertPosOrNull) {
-    return addDataLeafAt(insertPosOrNull.get());
-  } else {
-    return addDataLeafToFullTree();
-  }
-}
-
-unique_ref<DataLeafNode> DataTree::addDataLeafAt(DataInnerNode *insertPos) {
-  auto new_leaf = _nodeStore->createNewLeafNode();
-  auto chain = createChainOfInnerNodes(insertPos->depth()-1, new_leaf.get());
-  insertPos->addChild(*chain);
-  return new_leaf;
-}
-
-optional_ownership_ptr<DataNode> DataTree::createChainOfInnerNodes(unsigned int num, DataNode *child) {
-  //TODO This function is implemented twice, once with optional_ownership_ptr, once with unique_ref. Redundancy!
-  optional_ownership_ptr<DataNode> chain = cpputils::WithoutOwnership<DataNode>(child);
-  for(unsigned int i=0; i<num; ++i) {
-    auto newnode = _nodeStore->createNewInnerNode(*chain);
-    chain = cpputils::WithOwnership<DataNode>(std::move(newnode));
-  }
-  return chain;
-}
-
-unique_ref<DataNode> DataTree::createChainOfInnerNodes(unsigned int num, unique_ref<DataNode> child) {
-  unique_ref<DataNode> chain = std::move(child);
-  for(unsigned int i=0; i<num; ++i) {
-    chain = _nodeStore->createNewInnerNode(*chain);
-  }
-  return chain;
-}
-
-DataInnerNode* DataTree::increaseTreeDepth(unsigned int levels) {
-  ASSERT(levels >= 1, "Parameter out of bounds: tried to increase tree depth by zero.");
-  auto copyOfOldRoot = _nodeStore->createNewNodeAsCopyFrom(*_rootNode);
-  auto chain = createChainOfInnerNodes(levels-1, copyOfOldRoot.get());
-  auto newRootNode = DataNode::convertToNewInnerNode(std::move(_rootNode), *chain);
-  DataInnerNode *result = newRootNode.get();
-  _rootNode = std::move(newRootNode);
-  return result;
-}
-
-unique_ref<DataLeafNode> DataTree::addDataLeafToFullTree() {
-  DataInnerNode *rootNode = increaseTreeDepth(1);
-  auto newLeaf = addDataLeafAt(rootNode);
-  return newLeaf;
-}
-
-const Key &DataTree::key() const {
-  return _rootNode->key();
 }
 
 void DataTree::flush() const {
@@ -133,124 +52,67 @@ void DataTree::flush() const {
 }
 
 unique_ref<DataNode> DataTree::releaseRootNode() {
+  unique_lock<shared_mutex> lock(_mutex); // Lock ensures that the root node is currently set (traversing unsets it temporarily)
   return std::move(_rootNode);
 }
 
 //TODO Test numLeaves(), for example also two configurations with same number of bytes but different number of leaves (last leaf has 0 bytes)
 uint32_t DataTree::numLeaves() const {
     shared_lock<shared_mutex> lock(_mutex);
-  return _numLeaves(*_rootNode);
+    return _numLeaves();
 }
 
-uint32_t DataTree::_numLeaves(const DataNode &node) const {
+uint32_t DataTree::_numLeaves() const {
+  if (_numLeavesCache == none) {
+    _numLeavesCache = _computeNumLeaves(*_rootNode);
+  }
+  return *_numLeavesCache;
+}
+
+uint32_t DataTree::_forceComputeNumLeaves() const {
+  unique_lock<shared_mutex> lock(_mutex); // Lock ensures that the root node is currently set (traversing unsets it temporarily)
+  _numLeavesCache = _computeNumLeaves(*_rootNode);
+  return *_numLeavesCache;
+}
+
+uint32_t DataTree::_computeNumLeaves(const DataNode &node) const {
   const DataLeafNode *leaf = dynamic_cast<const DataLeafNode*>(&node);
   if (leaf != nullptr) {
     return 1;
   }
 
   const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(node);
-  uint64_t numLeavesInLeftChildren = (uint64_t)(inner.numChildren()-1) * leavesPerFullChild(inner);
+  uint64_t numLeavesInLeftChildren = static_cast<uint64_t>(inner.numChildren()-1) * leavesPerFullChild(inner);
-  auto lastChild = _nodeStore->load(inner.LastChild()->key());
+  auto lastChild = _nodeStore->load(inner.readLastChild().blockId());
   ASSERT(lastChild != none, "Couldn't load last child");
-  uint64_t numLeavesInRightChild = _numLeaves(**lastChild);
+  uint64_t numLeavesInRightChild = _computeNumLeaves(**lastChild);
 
   return numLeavesInLeftChildren + numLeavesInRightChild;
 }
 
-void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<void (DataLeafNode*, uint32_t)> func) {
+void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf) {
-  //TODO Can we traverse in parallel?
+  //TODO Can we allow multiple runs of traverseLeaves() in parallel? Also in parallel with resizeNumBytes()?
-  unique_lock<shared_mutex> lock(_mutex); //TODO Only lock when resizing. Otherwise parallel read/write to a blob is not possible!
+  std::unique_lock<shared_mutex> lock(_mutex);
   ASSERT(beginIndex <= endIndex, "Invalid parameters");
-  if (0 == endIndex) {
-    // In this case the utils::ceilLog(_, endIndex) below would fail
-    return;
-  }
 
-  uint8_t neededTreeDepth = utils::ceilLog(_nodeStore->layout().maxChildrenPerInnerNode(), (uint64_t)endIndex);
+  auto onBacktrackFromSubtree = [] (DataInnerNode* /*node*/) {};
-  uint32_t numLeaves = this->_numLeaves(*_rootNode); // TODO Querying the size causes a tree traversal down to the leaves. Possible without querying the size?
-  if (_rootNode->depth() < neededTreeDepth) {
-    //TODO Test cases that actually increase it here by 0 level / 1 level / more than 1 level
-    increaseTreeDepth(neededTreeDepth - _rootNode->depth());
-  }
 
-  if (numLeaves <= beginIndex) {
+  _traverseLeaves(beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
-    //TODO Test cases with numLeaves < / >= beginIndex
+
-    // There is a gap between the current size and the begin of the traversal
+  if (_numLeavesCache != none && *_numLeavesCache < endIndex) {
-    return _traverseLeaves(_rootNode.get(), 0, numLeaves-1, endIndex, [beginIndex, numLeaves, &func, this](DataLeafNode* node, uint32_t index) {
+    _numLeavesCache = endIndex;
-      if (index >= beginIndex) {
-        func(node, index);
-      } else if (index == numLeaves - 1) {
-        // It is the old last leaf - resize it to maximum
-        node->resize(_nodeStore->layout().maxBytesPerLeaf());
-      }
-    });
-  } else if (numLeaves < endIndex) {
-    // We are starting traversal in the valid region, but traverse until after it (we grow new leaves)
-    return _traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, [numLeaves, &func, this] (DataLeafNode *node, uint32_t index) {
-      if (index == numLeaves - 1) {
-        // It is the old last leaf  - resize it to maximum
-        node->resize(_nodeStore->layout().maxBytesPerLeaf());
-      }
-      func(node, index);
-    });
-  } else {
-    //We are traversing entirely inside the valid region
-    _traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, func);
   }
 }
 
-void DataTree::_traverseLeaves(DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, function<void (DataLeafNode*, uint32_t)> func) {
+void DataTree::_traverseLeaves(uint32_t beginIndex, uint32_t endIndex,
-  DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root);
+    function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
-  if (leaf != nullptr) {
+    function<Data (uint32_t index)> onCreateLeaf,
-    ASSERT(beginIndex <= 1 && endIndex <= 1, "If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1.");
+    function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
-    if (beginIndex == 0 && endIndex == 1) {
+  _rootNode = LeafTraverser(_nodeStore).traverseAndReturnRoot(std::move(_rootNode), beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
-      func(leaf, leafOffset);
-    }
-    return;
-  }
-
-  DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root);
-  uint32_t leavesPerChild = leavesPerFullChild(*inner);
-  uint32_t beginChild = beginIndex/leavesPerChild;
-  uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild);
-  vector<unique_ref<DataNode>> children = getOrCreateChildren(inner, beginChild, endChild);
-
-  for (uint32_t childIndex = beginChild; childIndex < endChild; ++childIndex) {
-    uint32_t childOffset = childIndex * leavesPerChild;
-    uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset);
-    uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
-    auto child = std::move(children[childIndex-beginChild]);
-    _traverseLeaves(child.get(), leafOffset + childOffset, localBeginIndex, localEndIndex, func);
-  }
-}
-
-vector<unique_ref<DataNode>> DataTree::getOrCreateChildren(DataInnerNode *node, uint32_t begin, uint32_t end) {
-  vector<unique_ref<DataNode>> children;
-  children.reserve(end-begin);
-  for (uint32_t childIndex = begin; childIndex < std::min(node->numChildren(), end); ++childIndex) {
-    auto child = _nodeStore->load(node->getChild(childIndex)->key());
-    ASSERT(child != none, "Couldn't load child node");
-    children.emplace_back(std::move(*child));
-  }
-  for (uint32_t childIndex = node->numChildren(); childIndex < end; ++childIndex) {
-    //TODO This creates each child with one chain to one leaf only, and then on the next lower level it
-    //     has to create the children for the child. Would be faster to directly create full trees if necessary.
-    children.emplace_back(addChildTo(node));
-  }
-  ASSERT(children.size() == end-begin, "Number of children in the result is wrong");
-  return children;
-}
-
-unique_ref<DataNode> DataTree::addChildTo(DataInnerNode *node) {
-  auto new_leaf = _nodeStore->createNewLeafNode();
-  new_leaf->resize(_nodeStore->layout().maxBytesPerLeaf());
-  auto chain = createChainOfInnerNodes(node->depth()-1, std::move(new_leaf));
-  node->addChild(*chain);
-  return chain;
 }
 
 uint32_t DataTree::leavesPerFullChild(const DataInnerNode &root) const {
-  return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), (uint64_t)root.depth()-1);
+  return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), static_cast<uint64_t>(root.depth())-1);
 }
 
 uint64_t DataTree::numStoredBytes() const {
@@ -270,7 +132,7 @@ uint64_t DataTree::_numStoredBytes(const DataNode &root) const {
 
   const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(root);
   uint64_t numBytesInLeftChildren = (inner.numChildren()-1) * leavesPerFullChild(inner) * _nodeStore->layout().maxBytesPerLeaf();
-  auto lastChild = _nodeStore->load(inner.LastChild()->key());
+  auto lastChild = _nodeStore->load(inner.readLastChild().blockId());
   ASSERT(lastChild != none, "Couldn't load last child");
   uint64_t numBytesInRightChild = _numStoredBytes(**lastChild);
 
@@ -278,57 +140,49 @@ uint64_t DataTree::_numStoredBytes(const DataNode &root) const {
 }
 
 void DataTree::resizeNumBytes(uint64_t newNumBytes) {
-  //TODO Can we resize in parallel? Especially creating new blocks (i.e. encrypting them) is expensive and should be done in parallel.
+  std::unique_lock<shared_mutex> lock(_mutex); // TODO Multiple ones in parallel? Also in parallel with traverseLeaves()?
-  boost::upgrade_lock<shared_mutex> lock(_mutex);
+
-  {
-    boost::upgrade_to_unique_lock<shared_mutex> exclusiveLock(lock);
-    //TODO Faster implementation possible (no addDataLeaf()/removeLastDataLeaf() in a loop, but directly resizing)
-    LastLeaf(_rootNode.get())->resize(_nodeStore->layout().maxBytesPerLeaf());
-    uint64_t currentNumBytes = _numStoredBytes();
-    ASSERT(currentNumBytes % _nodeStore->layout().maxBytesPerLeaf() == 0, "The last leaf is not a max data leaf, although we just resized it to be one.");
-    uint32_t currentNumLeaves = currentNumBytes / _nodeStore->layout().maxBytesPerLeaf();
   uint32_t newNumLeaves = std::max(UINT64_C(1), utils::ceilDivision(newNumBytes, _nodeStore->layout().maxBytesPerLeaf()));
-
-    for(uint32_t i = currentNumLeaves; i < newNumLeaves; ++i) {
-      addDataLeaf()->resize(_nodeStore->layout().maxBytesPerLeaf());
-    }
-    for(uint32_t i = currentNumLeaves; i > newNumLeaves; --i) {
-      removeLastDataLeaf();
-    }
   uint32_t newLastLeafSize = newNumBytes - (newNumLeaves-1) * _nodeStore->layout().maxBytesPerLeaf();
-    LastLeaf(_rootNode.get())->resize(newLastLeafSize);
+  uint32_t maxChildrenPerInnerNode = _nodeStore->layout().maxChildrenPerInnerNode();
+  auto onExistingLeaf = [newLastLeafSize] (uint32_t /*index*/, bool /*isRightBorderLeaf*/, LeafHandle leafHandle) {
+      auto leaf = leafHandle.node();
+      // This is only called, if the new last leaf was already existing
+      if (leaf->numBytes() != newLastLeafSize) {
+        leaf->resize(newLastLeafSize);
       }
-  ASSERT(newNumBytes == _numStoredBytes(), "We resized to the wrong number of bytes ("+std::to_string(numStoredBytes())+" instead of "+std::to_string(newNumBytes)+")");
+  };
+  auto onCreateLeaf = [newLastLeafSize] (uint32_t /*index*/) -> Data {
+      // This is only called, if the new last leaf was not existing yet
+      return Data(newLastLeafSize).FillWithZeroes();
+  };
+  auto onBacktrackFromSubtree = [this, newNumLeaves, maxChildrenPerInnerNode] (DataInnerNode* node) {
+      // This is only called for the right border nodes of the new tree.
+      // When growing size, the following is a no-op. When shrinking, we're deleting the children that aren't needed anymore.
+      uint32_t maxLeavesPerChild = utils::intPow(static_cast<uint64_t>(maxChildrenPerInnerNode), (static_cast<uint64_t>(node->depth())-1));
+      uint32_t neededNodesOnChildLevel = utils::ceilDivision(newNumLeaves, maxLeavesPerChild);
+      uint32_t neededSiblings = utils::ceilDivision(neededNodesOnChildLevel, maxChildrenPerInnerNode);
+      uint32_t neededChildrenForRightBorderNode = neededNodesOnChildLevel - (neededSiblings-1) * maxChildrenPerInnerNode;
+      ASSERT(neededChildrenForRightBorderNode <= node->numChildren(), "Node has too few children");
+      // All children to the right of the new right-border-node are removed including their subtree.
+      while(node->numChildren() > neededChildrenForRightBorderNode) {
+        _nodeStore->removeSubtree(node->depth()-1, node->readLastChild().blockId());
+        node->removeLastChild();
       }
+  };
 
-optional_ownership_ptr<DataLeafNode> DataTree::LastLeaf(DataNode *root) {
+  _traverseLeaves(newNumLeaves - 1, newNumLeaves, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
-  DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root);
+  _numLeavesCache = newNumLeaves;
-  if (leaf != nullptr) {
-    return WithoutOwnership(leaf);
-  }
-
-  DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root);
-  auto lastChild = _nodeStore->load(inner->LastChild()->key());
-  ASSERT(lastChild != none, "Couldn't load last child");
-  return WithOwnership(LastLeaf(std::move(*lastChild)));
-}
-
-unique_ref<DataLeafNode> DataTree::LastLeaf(unique_ref<DataNode> root) {
-  auto leaf = dynamic_pointer_move<DataLeafNode>(root);
-  if (leaf != none) {
-    return std::move(*leaf);
-  }
-  auto inner = dynamic_pointer_move<DataInnerNode>(root);
-  ASSERT(inner != none, "Root node is neither a leaf nor an inner node");
-  auto child = _nodeStore->load((*inner)->LastChild()->key());
-  ASSERT(child != none, "Couldn't load last child");
-  return LastLeaf(std::move(*child));
 }
 
 uint64_t DataTree::maxBytesPerLeaf() const {
   return _nodeStore->layout().maxBytesPerLeaf();
 }
 
+uint8_t DataTree::depth() const {
+  return _rootNode->depth();
+}
+
 }
 }
 }

src/blobstore/implementations/onblocks/datatreestore/DataTree.h

@@ -8,7 +8,8 @@
 #include "../datanodestore/DataNodeView.h"
 //TODO Replace with C++14 once std::shared_mutex is supported
 #include <boost/thread/shared_mutex.hpp>
-#include <blockstore/utils/Key.h>
+#include <blockstore/utils/BlockId.h>
+#include "LeafHandle.h"
 
 namespace blobstore {
 namespace onblocks {
@@ -26,48 +27,43 @@ public:
   DataTree(datanodestore::DataNodeStore *nodeStore, cpputils::unique_ref<datanodestore::DataNode> rootNode);
   ~DataTree();
 
-  const blockstore::Key &key() const;
+  const blockstore::BlockId &blockId() const;
   //Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
   uint64_t maxBytesPerLeaf() const;
 
-  void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func);
+  void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf);
   void resizeNumBytes(uint64_t newNumBytes);
 
   uint32_t numLeaves() const;
   uint64_t numStoredBytes() const;
 
+  uint8_t depth() const;
+
+  // only used by test cases
+  uint32_t _forceComputeNumLeaves() const;
+
   void flush() const;
 
 private:
   mutable boost::shared_mutex _mutex;
   datanodestore::DataNodeStore *_nodeStore;
   cpputils::unique_ref<datanodestore::DataNode> _rootNode;
-
+  blockstore::BlockId _blockId; // BlockId is stored in a member variable, since _rootNode is nullptr while traversing, but we still want to be able to return the blockId.
-  cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeaf();
+  mutable boost::optional<uint32_t> _numLeavesCache;
-  void removeLastDataLeaf();
 
   cpputils::unique_ref<datanodestore::DataNode> releaseRootNode();
   friend class DataTreeStore;
 
-  cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeafAt(datanodestore::DataInnerNode *insertPos);
-  cpputils::optional_ownership_ptr<datanodestore::DataNode> createChainOfInnerNodes(unsigned int num, datanodestore::DataNode *child);
-  cpputils::unique_ref<datanodestore::DataNode> createChainOfInnerNodes(unsigned int num, cpputils::unique_ref<datanodestore::DataNode> child);
-  cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeafToFullTree();
-
-  void deleteLastChildSubtree(datanodestore::DataInnerNode *node);
-  void ifRootHasOnlyOneChildReplaceRootWithItsChild();
-
   //TODO Use underscore for private methods
-  void _traverseLeaves(datanodestore::DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func);
+  void _traverseLeaves(uint32_t beginIndex, uint32_t endIndex,
+                       std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
+                       std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
+                       std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
   uint32_t leavesPerFullChild(const datanodestore::DataInnerNode &root) const;
   uint64_t _numStoredBytes() const;
   uint64_t _numStoredBytes(const datanodestore::DataNode &root) const;
-  uint32_t _numLeaves(const datanodestore::DataNode &node) const;
+  uint32_t _numLeaves() const;
-  cpputils::optional_ownership_ptr<datanodestore::DataLeafNode> LastLeaf(datanodestore::DataNode *root);
+  uint32_t _computeNumLeaves(const datanodestore::DataNode &node) const;
-  cpputils::unique_ref<datanodestore::DataLeafNode> LastLeaf(cpputils::unique_ref<datanodestore::DataNode> root);
-  datanodestore::DataInnerNode* increaseTreeDepth(unsigned int levels);
-  std::vector<cpputils::unique_ref<datanodestore::DataNode>> getOrCreateChildren(datanodestore::DataInnerNode *node, uint32_t begin, uint32_t end);
-  cpputils::unique_ref<datanodestore::DataNode> addChildTo(datanodestore::DataInnerNode *node);
 
   DISALLOW_COPY_AND_ASSIGN(DataTree);
 };

src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.cpp

@@ -1,15 +1,14 @@
 #include "DataTreeStore.h"
-#include "../datanodestore/DataNodeStore.h"
 #include "../datanodestore/DataLeafNode.h"
 #include "DataTree.h"
 
 using cpputils::unique_ref;
 using cpputils::make_unique_ref;
+using cpputils::Data;
 using boost::optional;
 using boost::none;
 
 using blobstore::onblocks::datanodestore::DataNodeStore;
-using blobstore::onblocks::datanodestore::DataNode;
 
 namespace blobstore {
 namespace onblocks {
@@ -22,8 +21,8 @@ DataTreeStore::DataTreeStore(unique_ref<DataNodeStore> nodeStore)
 DataTreeStore::~DataTreeStore() {
 }
 
-optional<unique_ref<DataTree>> DataTreeStore::load(const blockstore::Key &key) {
+optional<unique_ref<DataTree>> DataTreeStore::load(const blockstore::BlockId &blockId) {
-  auto node = _nodeStore->load(key);
+  auto node = _nodeStore->load(blockId);
   if (node == none) {
     return none;
   }
@@ -31,14 +30,18 @@ optional<unique_ref<DataTree>> DataTreeStore::load(const blockstore::Key &key) {
 }
 
 unique_ref<DataTree> DataTreeStore::createNewTree() {
-  auto newleaf = _nodeStore->createNewLeafNode();
+  auto newleaf = _nodeStore->createNewLeafNode(Data(0));
   return make_unique_ref<DataTree>(_nodeStore.get(), std::move(newleaf));
 }
 
 void DataTreeStore::remove(unique_ref<DataTree> tree) {
-  auto root = tree->releaseRootNode();
+  _nodeStore->removeSubtree(tree->releaseRootNode());
-  cpputils::destruct(std::move(tree)); // Destruct tree
+}
-  _nodeStore->removeSubtree(std::move(root));
+
+void DataTreeStore::remove(const blockstore::BlockId &blockId) {
+  auto tree = load(blockId);
+  ASSERT(tree != none, "Tree to remove not found");
+  remove(std::move(*tree));
 }
 
 }

src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h

@@ -5,7 +5,7 @@
 #include <memory>
 #include <cpp-utils/macros.h>
 #include <cpp-utils/pointer/unique_ref.h>
-#include <blockstore/utils/Key.h>
+#include <blockstore/utils/BlockId.h>
 #include <boost/optional.hpp>
 #include "../datanodestore/DataNodeStore.h"
 
@@ -19,11 +19,12 @@ public:
   DataTreeStore(cpputils::unique_ref<datanodestore::DataNodeStore> nodeStore);
   ~DataTreeStore();
 
-  boost::optional<cpputils::unique_ref<DataTree>> load(const blockstore::Key &key);
+  boost::optional<cpputils::unique_ref<DataTree>> load(const blockstore::BlockId &blockId);
 
   cpputils::unique_ref<DataTree> createNewTree();
 
   void remove(cpputils::unique_ref<DataTree> tree);
+  void remove(const blockstore::BlockId &blockId);
 
   //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
   uint64_t virtualBlocksizeBytes() const;

src/blobstore/implementations/onblocks/datatreestore/LeafHandle.cpp

@@ -0,0 +1,40 @@
+#include "LeafHandle.h"
+#include "../datanodestore/DataLeafNode.h"
+#include "../datanodestore/DataNodeStore.h"
+
+using cpputils::WithOwnership;
+using cpputils::WithoutOwnership;
+using boost::none;
+using cpputils::dynamic_pointer_move;
+using blobstore::onblocks::datanodestore::DataLeafNode;
+using blobstore::onblocks::datanodestore::DataNodeStore;
+using blockstore::BlockId;
+
+namespace blobstore {
+    namespace onblocks {
+        namespace datatreestore {
+
+            LeafHandle::LeafHandle(DataNodeStore *nodeStore, const BlockId &blockId)
+                    : _nodeStore(nodeStore), _blockId(blockId), _leaf(cpputils::null<DataLeafNode>()) {
+            }
+
+            LeafHandle::LeafHandle(DataNodeStore *nodeStore, DataLeafNode *node)
+                    : _nodeStore(nodeStore), _blockId(node->blockId()),
+                      _leaf(WithoutOwnership<DataLeafNode>(node)) {
+            }
+
+            DataLeafNode *LeafHandle::node() {
+                if (_leaf.get() == nullptr) {
+                    auto loaded = _nodeStore->load(_blockId);
+                    ASSERT(loaded != none, "Leaf not found");
+                    auto leaf = dynamic_pointer_move<DataLeafNode>(*loaded);
+                    ASSERT(leaf != none, "Loaded leaf is not leaf node");
+
+                    _leaf = WithOwnership(std::move(*leaf));
+                }
+
+                return _leaf.get();
+            }
+        }
+    }
+}

src/blobstore/implementations/onblocks/datatreestore/LeafHandle.h

@@ -0,0 +1,46 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_LEAFHANDLE_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_LEAFHANDLE_H_
+
+#include <cpp-utils/macros.h>
+#include <cpp-utils/pointer/optional_ownership_ptr.h>
+#include <blockstore/utils/BlockId.h>
+
+namespace blobstore {
+    namespace onblocks {
+        namespace datanodestore {
+            class DataNodeStore;
+            class DataLeafNode;
+        }
+        namespace datatreestore {
+
+            class LeafHandle final {
+            public:
+                LeafHandle(datanodestore::DataNodeStore *nodeStore, const blockstore::BlockId &blockId);
+                LeafHandle(datanodestore::DataNodeStore *nodeStore, datanodestore::DataLeafNode *node);
+                LeafHandle(LeafHandle &&rhs) = default;
+
+                const blockstore::BlockId &blockId() {
+                    return _blockId;
+                }
+
+                datanodestore::DataLeafNode *node();
+
+                datanodestore::DataNodeStore *nodeStore() {
+                    return _nodeStore;
+                }
+
+            private:
+                datanodestore::DataNodeStore *_nodeStore;
+                blockstore::BlockId _blockId;
+                cpputils::optional_ownership_ptr<datanodestore::DataLeafNode> _leaf;
+
+                DISALLOW_COPY_AND_ASSIGN(LeafHandle);
+            };
+
+
+        }
+    }
+}
+
+#endif

src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.cpp

@@ -0,0 +1,250 @@
+#include "LeafTraverser.h"
+#include <cpp-utils/assert/assert.h>
+#include "../../datanodestore/DataLeafNode.h"
+#include "../../datanodestore/DataInnerNode.h"
+#include "../../datanodestore/DataNodeStore.h"
+#include "../../utils/Math.h"
+
+using std::function;
+using std::vector;
+using boost::none;
+using cpputils::Data;
+using cpputils::unique_ref;
+using cpputils::dynamic_pointer_move;
+using blobstore::onblocks::datanodestore::DataNodeStore;
+using blobstore::onblocks::datanodestore::DataNode;
+using blobstore::onblocks::datanodestore::DataInnerNode;
+using blobstore::onblocks::datanodestore::DataLeafNode;
+
+namespace blobstore {
+    namespace onblocks {
+        namespace datatreestore {
+
+            LeafTraverser::LeafTraverser(DataNodeStore *nodeStore)
+                : _nodeStore(nodeStore) {
+            }
+
+            unique_ref<DataNode> LeafTraverser::traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
+                return _traverseAndReturnRoot(std::move(root), beginIndex, endIndex, true, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
+            }
+
+            unique_ref<DataNode> LeafTraverser::_traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
+                ASSERT(beginIndex <= endIndex, "Invalid parameters");
+
+                //TODO Test cases with numLeaves < / >= beginIndex, ideally test all configurations:
+                //     beginIndex<endIndex<numLeaves, beginIndex=endIndex<numLeaves, beginIndex<endIndex=numLeaves, beginIndex=endIndex=numLeaves
+                //     beginIndex<numLeaves<endIndex, beginIndex=numLeaves<endIndex,
+                //     numLeaves<beginIndex<endIndex, numLeaves<beginIndex=endIndex
+
+                uint32_t maxLeavesForDepth = _maxLeavesForTreeDepth(root->depth());
+                bool increaseTreeDepth = endIndex > maxLeavesForDepth;
+
+                if (root->depth() == 0) {
+                    DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root.get());
+                    ASSERT(leaf != nullptr, "Depth 0 has to be leaf node");
+
+                    if (increaseTreeDepth && leaf->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) {
+                        leaf->resize(_nodeStore->layout().maxBytesPerLeaf());
+                    }
+                    if (beginIndex == 0 && endIndex >= 1) {
+                        bool isRightBorderLeaf = (endIndex == 1);
+                        onExistingLeaf(0, isRightBorderLeaf, LeafHandle(_nodeStore, leaf));
+                    }
+                } else {
+                    DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root.get());
+                    ASSERT(inner != nullptr, "Depth != 0 has to be leaf node");
+                    _traverseExistingSubtree(inner, std::min(beginIndex, maxLeavesForDepth),
+                                             std::min(endIndex, maxLeavesForDepth), 0, isLeftBorderOfTraversal, !increaseTreeDepth,
+                                             increaseTreeDepth, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
+                }
+
+                // If the traversal goes too far right for a tree this depth, increase tree depth by one and continue traversal.
+                // This is recursive, i.e. will be repeated if the tree is still not deep enough.
+                // We don't increase to the full needed tree depth in one step, because we want the traversal to go as far as possible
+                // and only then increase the depth - this causes the tree to be in consistent shape (balanced) for longer.
+                if (increaseTreeDepth) {
+                    // TODO Test cases that increase tree depth by 0, 1, 2, ... levels
+                    auto newRoot = _increaseTreeDepth(std::move(root));
+                    return _traverseAndReturnRoot(std::move(newRoot), std::max(beginIndex, maxLeavesForDepth), endIndex, false, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
+                } else {
+                    // Once we're done growing the tree and done with the traversal, we might have to decrease tree depth,
+                    // because the callbacks could have deleted nodes (this happens for example when shrinking the tree using a traversal).
+                    return _whileRootHasOnlyOneChildReplaceRootWithItsChild(std::move(root));
+                }
+            }
+
+            unique_ref<DataInnerNode> LeafTraverser::_increaseTreeDepth(unique_ref<DataNode> root) {
+                auto copyOfOldRoot = _nodeStore->createNewNodeAsCopyFrom(*root);
+                return DataNode::convertToNewInnerNode(std::move(root), _nodeStore->layout(), *copyOfOldRoot);
+            }
+
+            void LeafTraverser::_traverseExistingSubtree(const blockstore::BlockId &blockId, uint8_t depth, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool isRightBorderNode, bool growLastLeaf, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
+                if (depth == 0) {
+                    ASSERT(beginIndex <= 1 && endIndex <= 1,
+                           "If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1.");
+                    LeafHandle leafHandle(_nodeStore, blockId);
+                    if (growLastLeaf) {
+                        if (leafHandle.node()->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) {
+                            leafHandle.node()->resize(_nodeStore->layout().maxBytesPerLeaf());
+                        }
+                    }
+                    if (beginIndex == 0 && endIndex == 1) {
+                        onExistingLeaf(leafOffset, isRightBorderNode, std::move(leafHandle));
+                    }
+                } else {
+                    auto node = _nodeStore->load(blockId);
+                    if (node == none) {
+                        throw std::runtime_error("Couldn't find child node " + blockId.ToString());
+                    }
+
+                    auto inner = dynamic_pointer_move<DataInnerNode>(*node);
+                    ASSERT(inner != none, "Has to be either leaf or inner node");
+                    ASSERT((*inner)->depth() == depth, "Wrong depth given");
+                    _traverseExistingSubtree(inner->get(), beginIndex, endIndex, leafOffset, isLeftBorderOfTraversal,
+                                             isRightBorderNode, growLastLeaf, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
+                }
+            }
+
+            void LeafTraverser::_traverseExistingSubtree(DataInnerNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool isRightBorderNode, bool growLastLeaf, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
+                ASSERT(beginIndex <= endIndex, "Invalid parameters");
+
+                //TODO Call callbacks for different leaves in parallel.
+
+                uint32_t leavesPerChild = _maxLeavesForTreeDepth(root->depth()-1);
+                uint32_t beginChild = beginIndex/leavesPerChild;
+                uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild);
+                ASSERT(endChild <= _nodeStore->layout().maxChildrenPerInnerNode(), "Traversal region would need increasing the tree depth. This should have happened before calling this function.");
+                uint32_t numChildren = root->numChildren();
+                ASSERT(!growLastLeaf || endChild >= numChildren, "Can only grow last leaf if it exists");
+                bool shouldGrowLastExistingLeaf = growLastLeaf || endChild > numChildren;
+
+                // If we traverse outside of the valid region (i.e. usually would only traverse to new leaves and not to the last leaf),
+                // we still have to descend to the last old child to fill it with leaves and grow the last old leaf.
+                if (isLeftBorderOfTraversal && beginChild >= numChildren) {
+                    ASSERT(numChildren > 0, "Node doesn't have children.");
+                    auto childBlockId = root->readLastChild().blockId();
+                    uint32_t childOffset = (numChildren-1) * leavesPerChild;
+                    _traverseExistingSubtree(childBlockId, root->depth()-1, leavesPerChild, leavesPerChild, childOffset, true, false, true,
+                                             [] (uint32_t /*index*/, bool /*isRightBorderNode*/, LeafHandle /*leaf*/) {ASSERT(false, "We don't actually traverse any leaves.");},
+                                             [] (uint32_t /*index*/) -> Data {ASSERT(false, "We don't actually traverse any leaves.");},
+                                             [] (DataInnerNode* /*node*/) {ASSERT(false, "We don't actually traverse any leaves.");});
+                }
+
+                // Traverse existing children
+                for (uint32_t childIndex = beginChild; childIndex < std::min(endChild, numChildren); ++childIndex) {
+                    auto childBlockId = root->readChild(childIndex).blockId();
+                    uint32_t childOffset = childIndex * leavesPerChild;
+                    uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset);
+                    uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
+                    bool isFirstChild = (childIndex == beginChild);
+                    bool isLastExistingChild = (childIndex == numChildren - 1);
+                    bool isLastChild = isLastExistingChild && (numChildren == endChild);
+                    ASSERT(localEndIndex <= leavesPerChild, "We don't want the child to add a tree level because it doesn't have enough space for the traversal.");
+                    _traverseExistingSubtree(childBlockId, root->depth()-1, localBeginIndex, localEndIndex, leafOffset + childOffset, isLeftBorderOfTraversal && isFirstChild,
+                                             isRightBorderNode && isLastChild, shouldGrowLastExistingLeaf && isLastExistingChild, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
+                }
+
+                // Traverse new children (including gap children, i.e. children that are created but not traversed because they're to the right of the current size, but to the left of the traversal region)
+                for (uint32_t childIndex = numChildren; childIndex < endChild; ++childIndex) {
+                    uint32_t childOffset = childIndex * leavesPerChild;
+                    uint32_t localBeginIndex = std::min(leavesPerChild, utils::maxZeroSubtraction(beginIndex, childOffset));
+                    uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
+                    auto leafCreator = (childIndex >= beginChild) ? onCreateLeaf : _createMaxSizeLeaf();
+                    auto child = _createNewSubtree(localBeginIndex, localEndIndex, leafOffset + childOffset, root->depth() - 1, leafCreator, onBacktrackFromSubtree);
+                    root->addChild(*child);
+                }
+
+                // This is only a backtrack, if we actually visited a leaf here.
+                if (endIndex > beginIndex) {
+                    onBacktrackFromSubtree(root);
+                }
+            }
+
+            unique_ref<DataNode> LeafTraverser::_createNewSubtree(uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, uint8_t depth, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
+                ASSERT(beginIndex <= endIndex, "Invalid parameters");
+                if (0 == depth) {
+                    ASSERT(beginIndex <= 1 && endIndex == 1, "With depth 0, we can only traverse one or zero leaves (i.e. traverse one leaf or traverse a gap leaf).");
+                    auto leafCreator = (beginIndex
+                     == 0) ? onCreateLeaf : _createMaxSizeLeaf();
+                    return _nodeStore->createNewLeafNode(leafCreator(leafOffset));
+                }
+
+                uint8_t minNeededDepth = utils::ceilLog(_nodeStore->layout().maxChildrenPerInnerNode(), static_cast<uint64_t>(endIndex));
+                ASSERT(depth >= minNeededDepth, "Given tree depth doesn't fit given number of leaves to create.");
+                uint32_t leavesPerChild = _maxLeavesForTreeDepth(depth-1);
+                uint32_t beginChild = beginIndex/leavesPerChild;
+                uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild);
+
+                vector<blockstore::BlockId> children;
+                children.reserve(endChild);
+                // TODO Remove redundancy of following two for loops by using min/max for calculating the parameters of the recursive call.
+                // Create gap children (i.e. children before the traversal but after the current size)
+                for (uint32_t childIndex = 0; childIndex < beginChild; ++childIndex) {
+                    uint32_t childOffset = childIndex * leavesPerChild;
+                    auto child = _createNewSubtree(leavesPerChild, leavesPerChild, leafOffset + childOffset, depth - 1,
+                                                   [] (uint32_t /*index*/)->Data {ASSERT(false, "We're only creating gap leaves here, not traversing any.");},
+                                                   [] (DataInnerNode* /*node*/) {});
+                    ASSERT(child->depth() == depth-1, "Created child node has wrong depth");
+                    children.push_back(child->blockId());
+                }
+                // Create new children that are traversed
+                for(uint32_t childIndex = beginChild; childIndex < endChild; ++childIndex) {
+                    uint32_t childOffset = childIndex * leavesPerChild;
+                    uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset);
+                    uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
+                    auto child = _createNewSubtree(localBeginIndex, localEndIndex, leafOffset + childOffset, depth - 1, onCreateLeaf, onBacktrackFromSubtree);
+                    ASSERT(child->depth() == depth-1, "Created child node has wrong depth");
+                    children.push_back(child->blockId());
+                }
+
+                ASSERT(children.size() > 0, "No children created");
+                auto newNode = _nodeStore->createNewInnerNode(depth, children);
+
+                // This is only a backtrack, if we actually created a leaf here.
+                if (endIndex > beginIndex) {
+                    onBacktrackFromSubtree(newNode.get());
+                }
+                return std::move(newNode);
+            }
+
+            uint32_t LeafTraverser::_maxLeavesForTreeDepth(uint8_t depth) const {
+                return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), static_cast<uint64_t>(depth));
+            }
+
+            function<Data (uint32_t index)> LeafTraverser::_createMaxSizeLeaf() const {
+                uint64_t maxBytesPerLeaf = _nodeStore->layout().maxBytesPerLeaf();
+                return [maxBytesPerLeaf] (uint32_t /*index*/) -> Data {
+                   return Data(maxBytesPerLeaf).FillWithZeroes();
+                };
+            }
+
+            unique_ref<DataNode> LeafTraverser::_whileRootHasOnlyOneChildReplaceRootWithItsChild(unique_ref<DataNode> root) {
+                DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root.get());
+                if (inner != nullptr && inner->numChildren() == 1) {
+                    auto newRoot = _whileRootHasOnlyOneChildRemoveRootReturnChild(inner->readChild(0).blockId());
+                    auto result = _nodeStore->overwriteNodeWith(std::move(root), *newRoot);
+                    _nodeStore->remove(std::move(newRoot));
+                    return result;
+                } else {
+                    return root;
+                }
+            }
+
+            unique_ref<DataNode> LeafTraverser::_whileRootHasOnlyOneChildRemoveRootReturnChild(const blockstore::BlockId &blockId) {
+                auto current = _nodeStore->load(blockId);
+                ASSERT(current != none, "Node not found");
+                auto inner = dynamic_pointer_move<DataInnerNode>(*current);
+                if (inner == none) {
+                    return std::move(*current);
+                } else if ((*inner)->numChildren() == 1) {
+                    auto result = _whileRootHasOnlyOneChildRemoveRootReturnChild((*inner)->readChild(0).blockId());
+                    _nodeStore->remove(std::move(*inner));
+                    return result;
+                } else {
+                    return std::move(*inner);
+                }
+            }
+
+        }
+    }
+}

src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.h

@@ -0,0 +1,69 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_LEAFTRAVERSER_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_LEAFTRAVERSER_H_
+
+#include <cpp-utils/macros.h>
+#include <cpp-utils/pointer/unique_ref.h>
+#include <cpp-utils/data/Data.h>
+#include <blockstore/utils/BlockId.h>
+#include "blobstore/implementations/onblocks/datatreestore/LeafHandle.h"
+
+namespace blobstore {
+    namespace onblocks {
+        namespace datanodestore {
+            class DataNodeStore;
+            class DataNode;
+            class DataLeafNode;
+            class DataInnerNode;
+        }
+        namespace datatreestore {
+
+            /**
+             * LeafTraverser can create leaves if they don't exist yet (i.e. endIndex > numLeaves), but
+             * it cannot increase the tree depth. That is, the tree has to be deep enough to allow
+             * creating the number of leaves.
+             */
+            class LeafTraverser final {
+            public:
+                LeafTraverser(datanodestore::DataNodeStore *nodeStore);
+
+                cpputils::unique_ref<datanodestore::DataNode> traverseAndReturnRoot(
+                      cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex,
+                      std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
+                      std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
+                      std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
+
+            private:
+                datanodestore::DataNodeStore *_nodeStore;
+
+                cpputils::unique_ref<datanodestore::DataNode> _traverseAndReturnRoot(
+                      cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal,
+                      std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
+                      std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
+                      std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
+                void _traverseExistingSubtree(datanodestore::DataInnerNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool isRightBorderNode, bool growLastLeaf,
+                      std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
+                      std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
+                      std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
+                void _traverseExistingSubtree(const blockstore::BlockId &blockId, uint8_t depth, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool isRightBorderNode, bool growLastLeaf,
+                                              std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
+                                              std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
+                                              std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
+                cpputils::unique_ref<datanodestore::DataInnerNode> _increaseTreeDepth(cpputils::unique_ref<datanodestore::DataNode> root);
+                cpputils::unique_ref<datanodestore::DataNode> _createNewSubtree(uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, uint8_t depth,
+                                                                                std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
+                                                                                std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
+                uint32_t _maxLeavesForTreeDepth(uint8_t depth) const;
+                std::function<cpputils::Data (uint32_t index)> _createMaxSizeLeaf() const;
+                cpputils::unique_ref<datanodestore::DataNode> _whileRootHasOnlyOneChildReplaceRootWithItsChild(cpputils::unique_ref<datanodestore::DataNode> root);
+                cpputils::unique_ref<datanodestore::DataNode> _whileRootHasOnlyOneChildRemoveRootReturnChild(const blockstore::BlockId &blockId);
+
+                DISALLOW_COPY_AND_ASSIGN(LeafTraverser);
+            };
+
+        }
+    }
+}
+
+
+#endif

src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.cpp

@@ -1,6 +1,6 @@
 #include "algorithms.h"
 #include <cpp-utils/pointer/cast.h>
-#include <blockstore/utils/Key.h>
+#include <blockstore/utils/BlockId.h>
 
 #include "../../datanodestore/DataInnerNode.h"
 #include "../../datanodestore/DataNodeStore.h"
@@ -13,7 +13,7 @@ using cpputils::unique_ref;
 using blobstore::onblocks::datanodestore::DataInnerNode;
 using blobstore::onblocks::datanodestore::DataNode;
 using blobstore::onblocks::datanodestore::DataNodeStore;
-using blockstore::Key;
+using blockstore::BlockId;
 using boost::optional;
 using boost::none;
 
@@ -23,8 +23,8 @@ namespace datatreestore {
 namespace algorithms {
 
 optional<unique_ref<DataInnerNode>> getLastChildAsInnerNode(DataNodeStore *nodeStore, const DataInnerNode &node) {
-  Key key = node.LastChild()->key();
+  BlockId blockId = node.readLastChild().blockId();
-  auto lastChild = nodeStore->load(key);
+  auto lastChild = nodeStore->load(blockId);
   ASSERT(lastChild != none, "Couldn't load last child");
   return dynamic_pointer_move<DataInnerNode>(*lastChild);
 }
@@ -40,10 +40,12 @@ optional_ownership_ptr<DataInnerNode> GetLowestInnerRightBorderNodeWithCondition
     if (condition(*currentNode)) {
       result = std::move(currentNode);
     }
-    ASSERT(lastChild != none || static_cast<int>(i) == rootNode->depth()-1, "Couldn't get last child as inner node but we're not deep enough yet for the last child to be a leaf");
+    if (lastChild == none) {
-    if (lastChild != none) {
+        // lastChild is a leaf
-      currentNode = cpputils::WithOwnership(std::move(*lastChild));
+        ASSERT(static_cast<int>(i) == rootNode->depth()-1, "Couldn't get last child as inner node but we're not deep enough yet for the last child to be a leaf");
+        break;
     }
+    currentNode = cpputils::WithOwnership(std::move(*lastChild));
   }
 
   return result;

src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h

@@ -4,25 +4,26 @@
 
 #include <parallelaccessstore/ParallelAccessStore.h>
 #include "../datatreestore/DataTree.h"
+#include "blobstore/implementations/onblocks/datatreestore/LeafHandle.h"
 
 namespace blobstore {
 namespace onblocks {
 namespace parallelaccessdatatreestore {
 
-class DataTreeRef final: public parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::Key>::ResourceRefBase {
+class DataTreeRef final: public parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::BlockId>::ResourceRefBase {
 public:
   DataTreeRef(datatreestore::DataTree *baseTree): _baseTree(baseTree) {}
 
-  const blockstore::Key &key() const {
+  const blockstore::BlockId &blockId() const {
-    return _baseTree->key();
+    return _baseTree->blockId();
   }
 
   uint64_t maxBytesPerLeaf() const {
     return _baseTree->maxBytesPerLeaf();
   }
 
-  void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func) {
+  void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, bool isRightBorderLeaf, datatreestore::LeafHandle leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf) {
-    return _baseTree->traverseLeaves(beginIndex, endIndex, func);
+    return _baseTree->traverseLeaves(beginIndex, endIndex, onExistingLeaf, onCreateLeaf);
   }
 
   uint32_t numLeaves() const {

src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp

@@ -9,12 +9,11 @@ using cpputils::make_unique_ref;
 using boost::optional;
 
 using blobstore::onblocks::datatreestore::DataTreeStore;
-using blockstore::Key;
+using blockstore::BlockId;
 
 namespace blobstore {
 namespace onblocks {
 using datatreestore::DataTreeStore;
-using datatreestore::DataTree;
 namespace parallelaccessdatatreestore {
 
 //TODO Here and for other stores (DataTreeStore, ...): Make small functions inline
@@ -26,23 +25,26 @@ ParallelAccessDataTreeStore::ParallelAccessDataTreeStore(unique_ref<DataTreeStor
 ParallelAccessDataTreeStore::~ParallelAccessDataTreeStore() {
 }
 
-optional<unique_ref<DataTreeRef>> ParallelAccessDataTreeStore::load(const blockstore::Key &key) {
+optional<unique_ref<DataTreeRef>> ParallelAccessDataTreeStore::load(const blockstore::BlockId &blockId) {
-  return _parallelAccessStore.load(key);
+  return _parallelAccessStore.load(blockId);
 }
 
 unique_ref<DataTreeRef> ParallelAccessDataTreeStore::createNewTree() {
   auto dataTree = _dataTreeStore->createNewTree();
-  Key key = dataTree->key();
+  BlockId blockId = dataTree->blockId();
-  return _parallelAccessStore.add(key, std::move(dataTree));
+  return _parallelAccessStore.add(blockId, std::move(dataTree));  // NOLINT (workaround https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82481 )
 }
 
 void ParallelAccessDataTreeStore::remove(unique_ref<DataTreeRef> tree) {
-  Key key = tree->key();
+  BlockId blockId = tree->blockId();
-  return _parallelAccessStore.remove(key, std::move(tree));
+  return _parallelAccessStore.remove(blockId, std::move(tree));
+}
+
+void ParallelAccessDataTreeStore::remove(const BlockId &blockId) {
+  return _parallelAccessStore.remove(blockId);
 }
 
 
-
 }
 }
 }

src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h

@@ -4,7 +4,7 @@
 
 #include <memory>
 #include <cpp-utils/macros.h>
-#include <blockstore/utils/Key.h>
+#include <blockstore/utils/BlockId.h>
 #include <parallelaccessstore/ParallelAccessStore.h>
 #include "../datatreestore/DataTreeStore.h"
 
@@ -20,11 +20,12 @@ public:
   ParallelAccessDataTreeStore(cpputils::unique_ref<datatreestore::DataTreeStore> dataTreeStore);
   ~ParallelAccessDataTreeStore();
 
-  boost::optional<cpputils::unique_ref<DataTreeRef>> load(const blockstore::Key &key);
+  boost::optional<cpputils::unique_ref<DataTreeRef>> load(const blockstore::BlockId &blockId);
 
   cpputils::unique_ref<DataTreeRef> createNewTree();
 
   void remove(cpputils::unique_ref<DataTreeRef> tree);
+  void remove(const blockstore::BlockId &blockId);
 
   //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
   uint64_t virtualBlocksizeBytes() const;
@@ -33,7 +34,7 @@ public:
 
 private:
   cpputils::unique_ref<datatreestore::DataTreeStore> _dataTreeStore;
-  parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::Key> _parallelAccessStore;
+  parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::BlockId> _parallelAccessStore;
 
   DISALLOW_COPY_AND_ASSIGN(ParallelAccessDataTreeStore);
 };

src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.h

@@ -11,20 +11,24 @@ namespace blobstore {
 namespace onblocks {
 namespace parallelaccessdatatreestore {
 
-class ParallelAccessDataTreeStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore<datatreestore::DataTree, blockstore::Key> {
+class ParallelAccessDataTreeStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore<datatreestore::DataTree, blockstore::BlockId> {
 public:
   ParallelAccessDataTreeStoreAdapter(datatreestore::DataTreeStore *baseDataTreeStore)
-    :_baseDataTreeStore(std::move(baseDataTreeStore)) {
+    :_baseDataTreeStore(baseDataTreeStore) {
   }
 
-  boost::optional<cpputils::unique_ref<datatreestore::DataTree>> loadFromBaseStore(const blockstore::Key &key) override {
+  boost::optional<cpputils::unique_ref<datatreestore::DataTree>> loadFromBaseStore(const blockstore::BlockId &blockId) override {
-	  return _baseDataTreeStore->load(key);
+	  return _baseDataTreeStore->load(blockId);
   }
 
   void removeFromBaseStore(cpputils::unique_ref<datatreestore::DataTree> dataTree) override {
 	  return _baseDataTreeStore->remove(std::move(dataTree));
   }
 
+  void removeFromBaseStore(const blockstore::BlockId &blockId) override {
+    return _baseDataTreeStore->remove(blockId);
+  }
+
 private:
   datatreestore::DataTreeStore *_baseDataTreeStore;
 

src/blobstore/implementations/onblocks/utils/Math.h

@@ -33,7 +33,7 @@ inline INT_TYPE maxZeroSubtraction(INT_TYPE minuend, INT_TYPE subtrahend) {
 
 template<typename INT_TYPE>
 inline INT_TYPE ceilLog(INT_TYPE base, INT_TYPE value) {
-    return std::ceil((long double)std::log(value)/(long double)std::log(base));
+    return std::ceil(static_cast<long double>(std::log(value))/static_cast<long double>(std::log(base)));
 }
 
 }

src/blobstore/interface/Blob.h

@@ -4,7 +4,7 @@
 
 #include <cstring>
 #include <cstdint>
-#include <blockstore/utils/Key.h>
+#include <blockstore/utils/BlockId.h>
 #include <cpp-utils/data/Data.h>
 
 namespace blobstore {
@@ -13,8 +13,8 @@ class Blob {
 public:
   virtual ~Blob() {}
 
-  //TODO Use own Key class for blobstore
+  //TODO Use own Id class for blobstore
-  virtual const blockstore::Key &key() const = 0;
+  virtual const blockstore::BlockId &blockId() const = 0;
 
   virtual uint64_t size() const = 0;
   virtual void resize(uint64_t numBytes) = 0;

src/blobstore/interface/BlobStore.h

@@ -6,7 +6,7 @@
 #include <string>
 #include <memory>
 
-#include <blockstore/utils/Key.h>
+#include <blockstore/utils/BlockId.h>
 #include <cpp-utils/pointer/unique_ref.h>
 
 namespace blobstore {
@@ -17,8 +17,9 @@ public:
   virtual ~BlobStore() {}
 
   virtual cpputils::unique_ref<Blob> create() = 0;
-  virtual boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::Key &key) = 0;
+  virtual boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::BlockId &blockId) = 0;
   virtual void remove(cpputils::unique_ref<Blob> blob) = 0;
+  virtual void remove(const blockstore::BlockId &blockId) = 0;
 
   virtual uint64_t numBlocks() const = 0;
   virtual uint64_t estimateSpaceForNumBlocksLeft() const = 0;

src/blockstore/CMakeLists.txt

@@ -1,14 +1,13 @@
 project (blockstore)
 
 set(SOURCES
-  utils/Key.cpp
+  utils/BlockId.cpp
+  utils/IdWrapper.cpp
   utils/BlockStoreUtils.cpp
   utils/FileDoesntExistException.cpp
-  interface/helpers/BlockStoreWithRandomKeys.cpp
   implementations/testfake/FakeBlockStore.cpp
   implementations/testfake/FakeBlock.cpp
-  implementations/inmemory/InMemoryBlock.cpp
+  implementations/inmemory/InMemoryBlockStore2.cpp
-  implementations/inmemory/InMemoryBlockStore.cpp
   implementations/parallelaccess/ParallelAccessBlockStore.cpp
   implementations/parallelaccess/BlockRef.cpp
   implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp
@@ -16,17 +15,21 @@ set(SOURCES
   implementations/compressing/CompressedBlock.cpp
   implementations/compressing/compressors/RunLengthEncoding.cpp
   implementations/compressing/compressors/Gzip.cpp
-  implementations/encrypted/EncryptedBlockStore.cpp
+  implementations/encrypted/EncryptedBlockStore2.cpp
-  implementations/encrypted/EncryptedBlock.cpp
+  implementations/ondisk/OnDiskBlockStore2.cpp
-  implementations/ondisk/OnDiskBlockStore.cpp
+  implementations/caching/CachingBlockStore2.cpp
-  implementations/ondisk/OnDiskBlock.cpp
-  implementations/caching/CachingBlockStore.cpp
   implementations/caching/cache/PeriodicTask.cpp
   implementations/caching/cache/CacheEntry.cpp
   implementations/caching/cache/Cache.cpp
   implementations/caching/cache/QueueMap.cpp
-  implementations/caching/CachedBlock.cpp
+  implementations/low2highlevel/LowToHighLevelBlock.cpp
-  implementations/caching/NewBlock.cpp
+  implementations/low2highlevel/LowToHighLevelBlockStore.cpp
+  implementations/integrity/IntegrityBlockStore2.cpp
+  implementations/integrity/KnownBlockVersions.cpp
+  implementations/integrity/ClientIdAndBlockId.cpp
+  implementations/integrity/IntegrityViolationError.cpp
+  implementations/mock/MockBlockStore.cpp
+  implementations/mock/MockBlock.cpp
 )
 
 add_library(${PROJECT_NAME} STATIC ${SOURCES})

src/blockstore/implementations/caching/CachedBlock.cpp

@@ -1,46 +0,0 @@
-#include "CachedBlock.h"
-#include "CachingBlockStore.h"
-
-using cpputils::unique_ref;
-
-namespace blockstore {
-namespace caching {
-
-CachedBlock::CachedBlock(unique_ref<Block> baseBlock, CachingBlockStore *blockStore)
-    :Block(baseBlock->key()),
-     _blockStore(blockStore),
-     _baseBlock(std::move(baseBlock)) {
-}
-
-CachedBlock::~CachedBlock() {
-  if (_baseBlock.get() != nullptr) {
-    _blockStore->release(std::move(_baseBlock));
-  }
-}
-
-const void *CachedBlock::data() const {
-  return _baseBlock->data();
-}
-
-void CachedBlock::write(const void *source, uint64_t offset, uint64_t size) {
-  return _baseBlock->write(source, offset, size);
-}
-
-void CachedBlock::flush() {
-  return _baseBlock->flush();
-}
-
-size_t CachedBlock::size() const {
-  return _baseBlock->size();
-}
-
-void CachedBlock::resize(size_t newSize) {
-    return _baseBlock->resize(newSize);
-}
-
-unique_ref<Block> CachedBlock::releaseBlock() {
-  return std::move(_baseBlock);
-}
-
-}
-}

src/blockstore/implementations/caching/CachedBlock.h

@@ -1,39 +0,0 @@
-#pragma once
-#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
-#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
-
-#include "../../interface/Block.h"
-
-#include <cpp-utils/pointer/unique_ref.h>
-
-namespace blockstore {
-namespace caching {
-class CachingBlockStore;
-
-class CachedBlock final: public Block {
-public:
-  //TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
-  CachedBlock(cpputils::unique_ref<Block> baseBlock, CachingBlockStore *blockStore);
-  ~CachedBlock();
-
-  const void *data() const override;
-  void write(const void *source, uint64_t offset, uint64_t size) override;
-  void flush() override;
-
-  size_t size() const override;
-
-  void resize(size_t newSize) override;
-
-  cpputils::unique_ref<Block> releaseBlock();
-
-private:
-  CachingBlockStore *_blockStore;
-  cpputils::unique_ref<Block> _baseBlock;
-
-  DISALLOW_COPY_AND_ASSIGN(CachedBlock);
-};
-
-}
-}
-
-#endif

src/blockstore/implementations/caching/CachingBlockStore.cpp

@@ -1,100 +0,0 @@
-#include "CachedBlock.h"
-#include "NewBlock.h"
-#include "CachingBlockStore.h"
-#include "../../interface/Block.h"
-
-#include <algorithm>
-#include <cpp-utils/pointer/cast.h>
-#include <cpp-utils/assert/assert.h>
-
-using cpputils::dynamic_pointer_move;
-using cpputils::Data;
-using boost::optional;
-using cpputils::unique_ref;
-using cpputils::make_unique_ref;
-using boost::none;
-
-namespace blockstore {
-namespace caching {
-
-CachingBlockStore::CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore)
-  :_baseBlockStore(std::move(baseBlockStore)), _cache(), _numNewBlocks(0) {
-}
-
-Key CachingBlockStore::createKey() {
-  return _baseBlockStore->createKey();
-}
-
-optional<unique_ref<Block>> CachingBlockStore::tryCreate(const Key &key, Data data) {
-  ASSERT(_cache.pop(key) == none, "Key already exists in cache");
-  //TODO Shouldn't we return boost::none if the key already exists?
-  //TODO Key can also already exist but not be in the cache right now.
-  ++_numNewBlocks;
-  return unique_ref<Block>(make_unique_ref<CachedBlock>(make_unique_ref<NewBlock>(key, std::move(data), this), this));
-}
-
-optional<unique_ref<Block>> CachingBlockStore::load(const Key &key) {
-  optional<unique_ref<Block>> optBlock = _cache.pop(key);
-  //TODO an optional<> class with .getOrElse() would make this code simpler. boost::optional<>::value_or_eval didn't seem to work with unique_ptr members.
-  if (optBlock != none) {
-    return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*optBlock), this));
-  } else {
-    auto block = _baseBlockStore->load(key);
-    if (block == none) {
-      return none;
-    } else {
-      return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*block), this));
-    }
-  }
-}
-
-void CachingBlockStore::remove(cpputils::unique_ref<Block> block) {
-  auto cached_block = dynamic_pointer_move<CachedBlock>(block);
-  ASSERT(cached_block != none, "Passed block is not a CachedBlock");
-  auto baseBlock = (*cached_block)->releaseBlock();
-  auto baseNewBlock = dynamic_pointer_move<NewBlock>(baseBlock);
-  if (baseNewBlock != none) {
-	if(!(*baseNewBlock)->alreadyExistsInBaseStore()) {
-	  --_numNewBlocks;
-	}
-    (*baseNewBlock)->remove();
-  } else {
-    _baseBlockStore->remove(std::move(baseBlock));
-  }
-}
-
-uint64_t CachingBlockStore::numBlocks() const {
-  return _baseBlockStore->numBlocks() + _numNewBlocks;
-}
-
-uint64_t CachingBlockStore::estimateNumFreeBytes() const {
-  return _baseBlockStore->estimateNumFreeBytes();
-}
-
-void CachingBlockStore::release(unique_ref<Block> block) {
-  Key key = block->key();
-  _cache.push(key, std::move(block));
-}
-
-optional<unique_ref<Block>> CachingBlockStore::tryCreateInBaseStore(const Key &key, Data data) {
-  auto block = _baseBlockStore->tryCreate(key, std::move(data));
-  if (block != none) {
-	--_numNewBlocks;
-  }
-  return block;
-}
-
-void CachingBlockStore::removeFromBaseStore(cpputils::unique_ref<Block> block) {
-  _baseBlockStore->remove(std::move(block));
-}
-
-void CachingBlockStore::flush() {
-  _cache.flush();
-}
-
-uint64_t CachingBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
-  return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
-}
-
-}
-}

src/blockstore/implementations/caching/CachingBlockStore.h

@@ -1,42 +0,0 @@
-#pragma once
-#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
-#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
-
-#include "cache/Cache.h"
-#include "../../interface/BlockStore.h"
-
-namespace blockstore {
-namespace caching {
-
-//TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel)
-class CachingBlockStore final: public BlockStore {
-public:
-  explicit CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
-
-  Key createKey() override;
-  boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
-  boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
-  void remove(cpputils::unique_ref<Block> block) override;
-  uint64_t numBlocks() const override;
-  uint64_t estimateNumFreeBytes() const override;
-  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
-
-  void release(cpputils::unique_ref<Block> block);
-
-  boost::optional<cpputils::unique_ref<Block>> tryCreateInBaseStore(const Key &key, cpputils::Data data);
-  void removeFromBaseStore(cpputils::unique_ref<Block> block);
-
-  void flush();
-
-private:
-  cpputils::unique_ref<BlockStore> _baseBlockStore;
-  Cache<Key, cpputils::unique_ref<Block>, 1000> _cache;
-  uint32_t _numNewBlocks;
-
-  DISALLOW_COPY_AND_ASSIGN(CachingBlockStore);
-};
-
-}
-}
-
-#endif

src/blockstore/implementations/caching/CachingBlockStore2.cpp

@@ -0,0 +1,155 @@
+#include "CachingBlockStore2.h"
+#include <memory>
+#include <cpp-utils/assert/assert.h>
+#include <cpp-utils/system/get_total_memory.h>
+
+using std::string;
+using std::mutex;
+using cpputils::Data;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using boost::optional;
+using boost::none;
+using std::unique_lock;
+using std::mutex;
+
+namespace blockstore {
+namespace caching {
+
+CachingBlockStore2::CachedBlock::CachedBlock(const CachingBlockStore2* blockStore, const BlockId &blockId, cpputils::Data data, bool isDirty)
+    : _blockStore(blockStore), _blockId(blockId), _data(std::move(data)), _dirty(isDirty) {
+}
+
+CachingBlockStore2::CachedBlock::~CachedBlock() {
+  if (_dirty) {
+    _blockStore->_baseBlockStore->store(_blockId, _data);
+  }
+  // remove it from the list of blocks not in the base store, if it's on it
+  unique_lock<mutex> lock(_blockStore->_cachedBlocksNotInBaseStoreMutex);
+  _blockStore->_cachedBlocksNotInBaseStore.erase(_blockId);
+}
+
+const Data& CachingBlockStore2::CachedBlock::read() const {
+  return _data;
+}
+
+void CachingBlockStore2::CachedBlock::markNotDirty() && {
+  _dirty = false; // Prevent writing it back into the base store
+}
+
+void CachingBlockStore2::CachedBlock::write(Data data) {
+  _data = std::move(data);
+  _dirty = true;
+}
+
+CachingBlockStore2::CachingBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore)
+: _baseBlockStore(std::move(baseBlockStore)), _cachedBlocksNotInBaseStoreMutex(), _cachedBlocksNotInBaseStore(), _cache() {
+}
+
+bool CachingBlockStore2::tryCreate(const BlockId &blockId, const Data &data) {
+  //TODO Check if block exists in base store? Performance hit? It's very unlikely it exists.
+  auto popped = _cache.pop(blockId);
+  if (popped != boost::none) {
+    // entry already exists in cache
+    _cache.push(blockId, std::move(*popped)); // push the just popped element back to the cache
+    return false;
+  } else {
+    _cache.push(blockId, make_unique_ref<CachingBlockStore2::CachedBlock>(this, blockId, data.copy(), true));
+    unique_lock<mutex> lock(_cachedBlocksNotInBaseStoreMutex);
+    _cachedBlocksNotInBaseStore.insert(blockId);
+    return true;
+  }
+}
+
+bool CachingBlockStore2::remove(const BlockId &blockId) {
+  // TODO Don't write-through but cache remove operations
+  auto popped = _cache.pop(blockId);
+  if (popped != boost::none) {
+    // Remove from base store if it exists in the base store
+    {
+      unique_lock<mutex> lock(_cachedBlocksNotInBaseStoreMutex);
+      if (_cachedBlocksNotInBaseStore.count(blockId) == 0) {
+          const bool existedInBaseStore = _baseBlockStore->remove(blockId);
+          if (!existedInBaseStore) {
+              throw std::runtime_error("Tried to remove block. Block existed in cache and stated it exists in base store, but wasn't found there.");
+          }
+      }
+    }
+    // Don't write back the cached block when it is destructed
+    std::move(**popped).markNotDirty();
+    return true;
+  } else {
+    return _baseBlockStore->remove(blockId);
+  }
+}
+
+optional<unique_ref<CachingBlockStore2::CachedBlock>> CachingBlockStore2::_loadFromCacheOrBaseStore(const BlockId &blockId) const {
+  auto popped = _cache.pop(blockId);
+  if (popped != boost::none) {
+    return std::move(*popped);
+  } else {
+    auto loaded = _baseBlockStore->load(blockId);
+    if (loaded == boost::none) {
+      return boost::none;
+    }
+    return make_unique_ref<CachingBlockStore2::CachedBlock>(this, blockId, std::move(*loaded), false);
+  }
+}
+
+optional<Data> CachingBlockStore2::load(const BlockId &blockId) const {
+  auto loaded = _loadFromCacheOrBaseStore(blockId);
+  if (loaded == boost::none) {
+    // TODO Cache non-existence?
+    return boost::none;
+  }
+  optional<Data> result = (*loaded)->read().copy();
+  _cache.push(blockId, std::move(*loaded));
+  return result;
+}
+
+void CachingBlockStore2::store(const BlockId &blockId, const Data &data) {
+  auto popped = _cache.pop(blockId);
+  if (popped != boost::none) {
+    (*popped)->write(data.copy());
+  } else {
+    popped = make_unique_ref<CachingBlockStore2::CachedBlock>(this, blockId, data.copy(), false);
+    // TODO Instead of storing it to the base store, we could just keep it dirty in the cache
+    //      and (if it doesn't exist in base store yet) add it to _cachedBlocksNotInBaseStore
+    _baseBlockStore->store(blockId, data);
+  }
+  _cache.push(blockId, std::move(*popped));
+}
+
+uint64_t CachingBlockStore2::numBlocks() const {
+  uint64_t numInCacheButNotInBaseStore = 0;
+  {
+    unique_lock<mutex> lock(_cachedBlocksNotInBaseStoreMutex);
+    numInCacheButNotInBaseStore = _cachedBlocksNotInBaseStore.size();
+  }
+  return _baseBlockStore->numBlocks() + numInCacheButNotInBaseStore;
+}
+
+uint64_t CachingBlockStore2::estimateNumFreeBytes() const {
+  return _baseBlockStore->estimateNumFreeBytes();
+}
+
+uint64_t CachingBlockStore2::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
+}
+
+void CachingBlockStore2::forEachBlock(std::function<void (const BlockId &)> callback) const {
+  {
+    unique_lock<mutex> lock(_cachedBlocksNotInBaseStoreMutex);
+    for (const BlockId &blockId : _cachedBlocksNotInBaseStore) {
+      callback(blockId);
+    }
+  }
+  _baseBlockStore->forEachBlock(std::move(callback));
+}
+
+void CachingBlockStore2::flush() {
+    _cache.flush();
+}
+
+}
+}

src/blockstore/implementations/caching/CachingBlockStore2.h

@@ -0,0 +1,63 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE2_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE2_H_
+
+#include "../../interface/BlockStore2.h"
+#include <cpp-utils/macros.h>
+#include "../caching/cache/Cache.h"
+#include <unordered_set>
+
+namespace blockstore {
+namespace caching {
+
+class CachingBlockStore2 final: public BlockStore2 {
+public:
+  CachingBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore);
+
+  bool tryCreate(const BlockId &blockId, const cpputils::Data &data) override;
+  bool remove(const BlockId &blockId) override;
+  boost::optional<cpputils::Data> load(const BlockId &blockId) const override;
+  void store(const BlockId &blockId, const cpputils::Data &data) override;
+  uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
+  void forEachBlock(std::function<void (const BlockId &)> callback) const override;
+
+  void flush();
+
+private:
+  // TODO Is a cache implementation with onEvict callback instead of destructor simpler?
+  class CachedBlock final {
+  public:
+    CachedBlock(const CachingBlockStore2* blockStore, const BlockId &blockId, cpputils::Data data, bool isDirty);
+    ~CachedBlock();
+
+    const cpputils::Data& read() const;
+    void write(cpputils::Data data);
+    void markNotDirty() &&; // only on rvalue because the destructor should be called after calling markNotDirty(). It shouldn't be put back into the cache.
+  private:
+    const CachingBlockStore2* _blockStore;
+    BlockId _blockId;
+    cpputils::Data _data;
+    bool _dirty;
+
+    DISALLOW_COPY_AND_ASSIGN(CachedBlock);
+  };
+
+  boost::optional<cpputils::unique_ref<CachedBlock>> _loadFromCacheOrBaseStore(const BlockId &blockId) const;
+
+  cpputils::unique_ref<BlockStore2> _baseBlockStore;
+  friend class CachedBlock;
+
+  // TODO Store CachedBlock directly, without unique_ref
+  mutable std::mutex _cachedBlocksNotInBaseStoreMutex;
+  mutable std::unordered_set<BlockId> _cachedBlocksNotInBaseStore;
+  mutable Cache<BlockId, cpputils::unique_ref<CachedBlock>, 1000> _cache;
+
+  DISALLOW_COPY_AND_ASSIGN(CachingBlockStore2);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/caching/NewBlock.cpp

@@ -1,75 +0,0 @@
-#include "NewBlock.h"
-#include "CachingBlockStore.h"
-#include <cpp-utils/assert/assert.h>
-#include <cpp-utils/data/DataUtils.h>
-
-using cpputils::Data;
-using boost::none;
-
-namespace blockstore {
-namespace caching {
-
-NewBlock::NewBlock(const Key &key, Data data, CachingBlockStore *blockStore)
-    :Block(key),
-     _blockStore(blockStore),
-     _data(std::move(data)),
-     _baseBlock(none),
-     _dataChanged(true) {
-}
-
-NewBlock::~NewBlock() {
-  writeToBaseBlockIfChanged();
-}
-
-const void *NewBlock::data() const {
-  return _data.data();
-}
-
-void NewBlock::write(const void *source, uint64_t offset, uint64_t size) {
-  ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area");
-  std::memcpy((uint8_t*)_data.data()+offset, source, size);
-  _dataChanged = true;
-}
-
-void NewBlock::writeToBaseBlockIfChanged() {
-  if (_dataChanged) {
-    if (_baseBlock == none) {
-      //TODO _data.copy() necessary?
-      auto newBase = _blockStore->tryCreateInBaseStore(key(), _data.copy());
-      ASSERT(newBase != boost::none, "Couldn't create base block"); //TODO What if tryCreate fails due to a duplicate key? We should ensure we don't use duplicate keys.
-      _baseBlock = std::move(*newBase);
-    } else {
-        (*_baseBlock)->write(_data.data(), 0, _data.size());
-    }
-	_dataChanged = false;
-  }
-}
-
-void NewBlock::remove() {
-  if (_baseBlock != none) {
-	_blockStore->removeFromBaseStore(std::move(*_baseBlock));
-  }
-  _dataChanged = false;
-}
-
-void NewBlock::flush() {
-  writeToBaseBlockIfChanged();
-  ASSERT(_baseBlock != none, "At this point, the base block should already have been created but wasn't");
-  (*_baseBlock)->flush();
-}
-
-size_t NewBlock::size() const {
-  return _data.size();
-}
-
-void NewBlock::resize(size_t newSize) {
-    _data = cpputils::DataUtils::resize(std::move(_data), newSize);
-    _dataChanged = true;
-}
-
-bool NewBlock::alreadyExistsInBaseStore() const {
-  return _baseBlock != none;
-}
-
-}
-}

src/blockstore/implementations/caching/NewBlock.h

@@ -1,52 +0,0 @@
-#pragma once
-#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
-#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
-
-#include "../../interface/BlockStore.h"
-#include <cpp-utils/data/Data.h>
-
-#include <cpp-utils/macros.h>
-#include <memory>
-
-namespace blockstore {
-namespace caching {
-class CachingBlockStore;
-
-//TODO Does it make sense to write a general DataBackedBlock that just stores a Data object and maps the block operations to it?
-//     Can we reuse that object somewhere else?
-//     Maybe a second abstract class for BlockRefBackedBlock?
-
-// This is a block that was created in CachingBlockStore, but doesn't exist in the base block store yet.
-// It only exists in the cache and it is created in the base block store when destructed.
-class NewBlock final: public Block {
-public:
-  NewBlock(const Key &key, cpputils::Data data, CachingBlockStore *blockStore);
-  ~NewBlock();
-
-  const void *data() const override;
-  void write(const void *source, uint64_t offset, uint64_t size) override;
-  void flush() override;
-
-  size_t size() const override;
-
-  void resize(size_t newSize) override;
-
-  void remove();
-
-  bool alreadyExistsInBaseStore() const;
-
-private:
-  CachingBlockStore *_blockStore;
-  cpputils::Data _data;
-  boost::optional<cpputils::unique_ref<Block>> _baseBlock;
-  bool _dataChanged;
-
-  void writeToBaseBlockIfChanged();
-
-  DISALLOW_COPY_AND_ASSIGN(NewBlock);
-};
-
-}
-}
-
-#endif

src/blockstore/implementations/caching/cache/Cache.h

@@ -132,7 +132,7 @@ void Cache<Key, Value, MAX_ENTRIES>::_deleteOldEntriesParallel() {
 template<class Key, class Value, uint32_t MAX_ENTRIES>
 void Cache<Key, Value, MAX_ENTRIES>::_deleteMatchingEntriesAtBeginningParallel(std::function<bool (const CacheEntry<Key, Value> &)> matches) {
   // Twice the number of cores, so we use full CPU even if half the threads are doing I/O
-  unsigned int numThreads = 2 * std::max(1u, std::thread::hardware_concurrency());
+  unsigned int numThreads = 2 * (std::max)(1u, std::thread::hardware_concurrency());
   std::vector<std::future<void>> waitHandles;
   for (unsigned int i = 0; i < numThreads; ++i) {
     waitHandles.push_back(std::async(std::launch::async, [this, matches] {

src/blockstore/implementations/caching/cache/CacheEntry.h

@@ -16,10 +16,10 @@ public:
   explicit CacheEntry(Value value): _lastAccess(currentTime()), _value(std::move(value)) {
   }
 
-  CacheEntry(CacheEntry &&) = default;
+  CacheEntry(CacheEntry&& rhs) noexcept: _lastAccess(std::move(rhs._lastAccess)), _value(std::move(rhs._value)) {}
 
   double ageSeconds() const {
-    return ((double)(currentTime() - _lastAccess).total_nanoseconds()) / ((double)1000000000);
+    return static_cast<double>((currentTime() - _lastAccess).total_nanoseconds()) / static_cast<double>(1000000000);
   }
 
   Value releaseValue() {

src/blockstore/implementations/caching/cache/PeriodicTask.cpp

@@ -2,7 +2,6 @@
 #include <cpp-utils/logging/logging.h>
 
 using std::function;
-using std::endl;
 using namespace cpputils::logging;
 
 namespace blockstore {
@@ -10,7 +9,7 @@ namespace caching {
 
 PeriodicTask::PeriodicTask(function<void ()> task, double intervalSec) :
         _task(task),
-        _interval((uint64_t)(UINT64_C(1000000000) * intervalSec)),
+        _interval(static_cast<uint64_t>(UINT64_C(1000000000) * intervalSec)),
         _thread(std::bind(&PeriodicTask::_loopIteration, this)) {
     _thread.start();
 }

src/blockstore/implementations/compressing/CompressedBlock.h

@@ -18,7 +18,8 @@ template<class Compressor> class CompressingBlockStore;
 template<class Compressor>
 class CompressedBlock final: public Block {
 public:
-  static boost::optional<cpputils::unique_ref<CompressedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData);
+  static boost::optional<cpputils::unique_ref<CompressedBlock>> TryCreateNew(BlockStore *baseBlockStore, const BlockId &blockId, cpputils::Data decompressedData);
+  static cpputils::unique_ref<CompressedBlock> Overwrite(BlockStore *baseBlockStore, const BlockId &blockId, cpputils::Data decompressedData);
   static cpputils::unique_ref<CompressedBlock> Decompress(cpputils::unique_ref<Block> baseBlock);
 
   CompressedBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data decompressedData);
@@ -46,9 +47,9 @@ private:
 };
 
 template<class Compressor>
-boost::optional<cpputils::unique_ref<CompressedBlock<Compressor>>> CompressedBlock<Compressor>::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData) {
+boost::optional<cpputils::unique_ref<CompressedBlock<Compressor>>> CompressedBlock<Compressor>::TryCreateNew(BlockStore *baseBlockStore, const BlockId &blockId, cpputils::Data decompressedData) {
   cpputils::Data compressed = Compressor::Compress(decompressedData);
-  auto baseBlock = baseBlockStore->tryCreate(key, std::move(compressed));
+  auto baseBlock = baseBlockStore->tryCreate(blockId, std::move(compressed));
   if (baseBlock == boost::none) {
     //TODO Test this code branch
     return boost::none;
@@ -57,15 +58,23 @@ boost::optional<cpputils::unique_ref<CompressedBlock<Compressor>>> CompressedBlo
   return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(*baseBlock), std::move(decompressedData));
 }
 
+template<class Compressor>
+cpputils::unique_ref<CompressedBlock<Compressor>> CompressedBlock<Compressor>::Overwrite(BlockStore *baseBlockStore, const BlockId &blockId, cpputils::Data decompressedData) {
+  cpputils::Data compressed = Compressor::Compress(decompressedData);
+  auto baseBlock = baseBlockStore->overwrite(blockId, std::move(compressed));
+
+  return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(baseBlock), std::move(decompressedData));
+}
+
 template<class Compressor>
 cpputils::unique_ref<CompressedBlock<Compressor>> CompressedBlock<Compressor>::Decompress(cpputils::unique_ref<Block> baseBlock) {
-  cpputils::Data decompressed = Compressor::Decompress((CryptoPP::byte*)baseBlock->data(), baseBlock->size());
+  cpputils::Data decompressed = Compressor::Decompress(baseBlock->data(), baseBlock->size());
   return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(baseBlock), std::move(decompressed));
 }
 
 template<class Compressor>
 CompressedBlock<Compressor>::CompressedBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data decompressedData)
-        : Block(baseBlock->key()),
+        : Block(baseBlock->blockId()),
           _baseBlock(std::move(baseBlock)),
           _decompressedData(std::move(decompressedData)),
           _dataChanged(false) {
@@ -84,7 +93,7 @@ const void *CompressedBlock<Compressor>::data() const {
 
 template<class Compressor>
 void CompressedBlock<Compressor>::write(const void *source, uint64_t offset, uint64_t size) {
-  std::memcpy((uint8_t*)_decompressedData.dataOffset(offset), source, size);
+  std::memcpy(_decompressedData.dataOffset(offset), source, size);
   _dataChanged = true;
 }
 
@@ -102,7 +111,7 @@ size_t CompressedBlock<Compressor>::size() const {
 
 template<class Compressor>
 void CompressedBlock<Compressor>::resize(size_t newSize) {
-  _decompressedData = cpputils::DataUtils::resize(std::move(_decompressedData), newSize);
+  _decompressedData = cpputils::DataUtils::resize(_decompressedData, newSize);
   _dataChanged = true;
 }
 

src/blockstore/implementations/compressing/CompressingBlockStore.h

@@ -14,13 +14,15 @@ public:
     CompressingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
     ~CompressingBlockStore();
 
-    Key createKey() override;
+    BlockId createBlockId() override;
-    boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
+    boost::optional<cpputils::unique_ref<Block>> tryCreate(const BlockId &blockId, cpputils::Data data) override;
-    boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
+    boost::optional<cpputils::unique_ref<Block>> load(const BlockId &blockId) override;
-    void remove(cpputils::unique_ref<Block> block) override;
+    cpputils::unique_ref<Block> overwrite(const blockstore::BlockId &blockId, cpputils::Data data) override;
+    void remove(const BlockId &blockId) override;
     uint64_t numBlocks() const override;
     uint64_t estimateNumFreeBytes() const override;
     uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
+    void forEachBlock(std::function<void (const BlockId &)> callback) const override;
 
 private:
     cpputils::unique_ref<BlockStore> _baseBlockStore;
@@ -38,13 +40,13 @@ CompressingBlockStore<Compressor>::~CompressingBlockStore() {
 }
 
 template<class Compressor>
-Key CompressingBlockStore<Compressor>::createKey() {
+BlockId CompressingBlockStore<Compressor>::createBlockId() {
-    return _baseBlockStore->createKey();
+    return _baseBlockStore->createBlockId();
 }
 
 template<class Compressor>
-boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::tryCreate(const Key &key, cpputils::Data data) {
+boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::tryCreate(const BlockId &blockId, cpputils::Data data) {
-    auto result = CompressedBlock<Compressor>::TryCreateNew(_baseBlockStore.get(), key, std::move(data));
+    auto result = CompressedBlock<Compressor>::TryCreateNew(_baseBlockStore.get(), blockId, std::move(data));
     if (result == boost::none) {
         return boost::none;
     }
@@ -52,8 +54,13 @@ boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::
 }
 
 template<class Compressor>
-boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::load(const Key &key) {
+cpputils::unique_ref<Block> CompressingBlockStore<Compressor>::overwrite(const blockstore::BlockId &blockId, cpputils::Data data) {
-    auto loaded = _baseBlockStore->load(key);
+    return CompressedBlock<Compressor>::Overwrite(_baseBlockStore.get(), blockId, std::move(data));
+}
+
+template<class Compressor>
+boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::load(const BlockId &blockId) {
+    auto loaded = _baseBlockStore->load(blockId);
     if (loaded == boost::none) {
         return boost::none;
     }
@@ -61,11 +68,8 @@ boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::
 }
 
 template<class Compressor>
-void CompressingBlockStore<Compressor>::remove(cpputils::unique_ref<Block> block) {
+void CompressingBlockStore<Compressor>::remove(const BlockId &blockId) {
-    auto _block = cpputils::dynamic_pointer_move<CompressedBlock<Compressor>>(block);
+    return _baseBlockStore->remove(blockId);
-    ASSERT(_block != boost::none, "Wrong block type");
-    auto baseBlock = (*_block)->releaseBaseBlock();
-    return _baseBlockStore->remove(std::move(baseBlock));
 }
 
 template<class Compressor>
@@ -78,6 +82,11 @@ uint64_t CompressingBlockStore<Compressor>::estimateNumFreeBytes() const {
     return _baseBlockStore->estimateNumFreeBytes();
 }
 
+template<class Compressor>
+void CompressingBlockStore<Compressor>::forEachBlock(std::function<void (const BlockId &)> callback) const {
+    return _baseBlockStore->forEachBlock(callback);
+}
+
 template<class Compressor>
 uint64_t CompressingBlockStore<Compressor>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
     //We probably have more since we're compressing, but we don't know exactly how much.

src/blockstore/implementations/compressing/compressors/Gzip.cpp

@@ -1,6 +1,6 @@
 #include "cpp-utils/crypto/cryptopp_byte.h"
 #include "Gzip.h"
-#include <cryptopp/gzip.h>
+#include <vendor_cryptopp/gzip.h>
 
 using cpputils::Data;
 
@@ -9,20 +9,20 @@ namespace blockstore {
 
         Data Gzip::Compress(const Data &data) {
             CryptoPP::Gzip zipper;
-            zipper.Put((CryptoPP::byte *) data.data(), data.size());
+            zipper.Put(static_cast<const CryptoPP::byte *>(data.data()), data.size());
             zipper.MessageEnd();
             Data compressed(zipper.MaxRetrievable());
-            zipper.Get((CryptoPP::byte *) compressed.data(), compressed.size());
+            zipper.Get(static_cast<CryptoPP::byte *>(compressed.data()), compressed.size());
             return compressed;
         }
 
         Data Gzip::Decompress(const void *data, size_t size) {
             //TODO Change interface to taking cpputils::Data objects (needs changing blockstore so we can read their "class Data", because this is called from CompressedBlock::Decompress()).
             CryptoPP::Gunzip zipper;
-            zipper.Put((CryptoPP::byte *) data, size);
+            zipper.Put(static_cast<const CryptoPP::byte *>(data), size);
             zipper.MessageEnd();
             Data decompressed(zipper.MaxRetrievable());
-            zipper.Get((CryptoPP::byte *) decompressed.data(), decompressed.size());
+            zipper.Get(static_cast<CryptoPP::byte *>(decompressed.data()), decompressed.size());
             return decompressed;
         }
 

src/blockstore/implementations/compressing/compressors/RunLengthEncoding.cpp

@@ -18,8 +18,8 @@ namespace blockstore {
 
         Data RunLengthEncoding::Compress(const Data &data) {
             ostringstream compressed;
-            uint8_t *current = (uint8_t*)data.data();
+            const uint8_t *current = static_cast<const uint8_t*>(data.data());
-            uint8_t *end = (uint8_t*)data.data()+data.size();
+            const uint8_t *end = static_cast<const uint8_t*>(data.data())+data.size();
             while (current < end) {
                 _encodeArbitraryWords(&current, end, &compressed);
                 ASSERT(current <= end, "Overflow");
@@ -32,25 +32,25 @@ namespace blockstore {
             return _extractData(&compressed);
         }
 
-        void RunLengthEncoding::_encodeArbitraryWords(uint8_t **current, uint8_t* end, ostringstream *output) {
+        void RunLengthEncoding::_encodeArbitraryWords(const uint8_t **current, const uint8_t* end, ostringstream *output) {
             uint16_t size = _arbitraryRunLength(*current, end);
-            output->write((const char*)&size, sizeof(uint16_t));
+            output->write(reinterpret_cast<const char*>(&size), sizeof(uint16_t));
-            output->write((const char*)*current, size);
+            output->write(reinterpret_cast<const char*>(*current), size);
             *current += size;
         }
 
-        uint16_t RunLengthEncoding::_arbitraryRunLength(uint8_t *start, uint8_t* end) {
+        uint16_t RunLengthEncoding::_arbitraryRunLength(const uint8_t *start, const uint8_t* end) {
             // Each stopping of an arbitrary bytes run costs us 5 byte, because we have to store the length
             // for the identical bytes run (2 byte), the identical byte itself (1 byte) and the length for the next arbitrary bytes run (2 byte).
             // So to get an advantage from stopping an arbitrary bytes run, at least 6 bytes have to be identical.
 
             // realEnd avoids an overflow of the 16bit counter
-            uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max());
+            const uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max());
 
             // Count the number of identical bytes and return if it finds a run of more than 6 identical bytes.
             uint8_t lastByte = *start + 1; // Something different from the first byte
             uint8_t numIdenticalBytes = 1;
-            for(uint8_t *current = start; current != realEnd; ++current) {
+            for(const uint8_t *current = start; current != realEnd; ++current) {
                 if (*current == lastByte) {
                     ++numIdenticalBytes;
                     if (numIdenticalBytes == 6) {
@@ -65,16 +65,16 @@ namespace blockstore {
             return realEnd-start;
         }
 
-        void RunLengthEncoding::_encodeIdenticalWords(uint8_t **current, uint8_t* end, ostringstream *output) {
+        void RunLengthEncoding::_encodeIdenticalWords(const uint8_t **current, const uint8_t* end, ostringstream *output) {
             uint16_t size = _countIdenticalBytes(*current, end);
-            output->write((const char*)&size, sizeof(uint16_t));
+            output->write(reinterpret_cast<const char*>(&size), sizeof(uint16_t));
-            output->write((const char*)*current, 1);
+            output->write(reinterpret_cast<const char*>(*current), 1);
             *current += size;
         }
 
-        uint16_t RunLengthEncoding::_countIdenticalBytes(uint8_t *start, uint8_t *end) {
+        uint16_t RunLengthEncoding::_countIdenticalBytes(const uint8_t *start, const uint8_t *end) {
-            uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max()); // This prevents overflow of the 16bit counter
+            const uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max()); // This prevents overflow of the 16bit counter
-            for (uint8_t *current = start+1; current != realEnd; ++current) {
+            for (const uint8_t *current = start+1; current != realEnd; ++current) {
                 if (*current != *start) {
                     return current-start;
                 }
@@ -92,7 +92,7 @@ namespace blockstore {
 
         Data RunLengthEncoding::Decompress(const void *data, size_t size) {
             istringstream stream;
-            _parseData((uint8_t*)data, size, &stream);
+            _parseData(static_cast<const uint8_t*>(data), size, &stream);
             ostringstream decompressed;
             while(_hasData(&stream)) {
                 _decodeArbitraryWords(&stream, &decompressed);
@@ -110,29 +110,29 @@ namespace blockstore {
         }
 
         void RunLengthEncoding::_parseData(const uint8_t *data, size_t size, istringstream *result) {
-            result->str(string((const char*)data, size));
+            result->str(string(reinterpret_cast<const char*>(data), size));
         }
 
         void RunLengthEncoding::_decodeArbitraryWords(istringstream *stream, ostringstream *decompressed) {
             uint16_t size;
-            stream->read((char*)&size, sizeof(uint16_t));
+            stream->read(reinterpret_cast<char*>(&size), sizeof(uint16_t));
             ASSERT(stream->good(), "Premature end of stream");
             Data run(size);
-            stream->read((char*)run.data(), size);
+            stream->read(static_cast<char*>(run.data()), size);
             ASSERT(stream->good(), "Premature end of stream");
-            decompressed->write((const char*)run.data(), run.size());
+            decompressed->write(static_cast<const char*>(run.data()), run.size());
         }
 
         void RunLengthEncoding::_decodeIdenticalWords(istringstream *stream, ostringstream *decompressed) {
             uint16_t size;
-            stream->read((char*)&size, sizeof(uint16_t));
+            stream->read(reinterpret_cast<char*>(&size), sizeof(uint16_t));
             ASSERT(stream->good(), "Premature end of stream");
             uint8_t value;
-            stream->read((char*)&value, 1);
+            stream->read(reinterpret_cast<char*>(&value), 1);
             ASSERT(stream->good(), "Premature end of stream");
             Data run(size);
             std::memset(run.data(), value, run.size());
-            decompressed->write((const char*)run.data(), run.size());
+            decompressed->write(static_cast<const char*>(run.data()), run.size());
         }
 
     }

src/blockstore/implementations/compressing/compressors/RunLengthEncoding.h

@@ -13,10 +13,10 @@ namespace blockstore {
             static cpputils::Data Decompress(const void *data, size_t size);
 
         private:
-            static void _encodeArbitraryWords(uint8_t **current, uint8_t* end, std::ostringstream *output);
+            static void _encodeArbitraryWords(const uint8_t **current, const uint8_t* end, std::ostringstream *output);
-            static uint16_t _arbitraryRunLength(uint8_t *start, uint8_t* end);
+            static uint16_t _arbitraryRunLength(const uint8_t *start, const uint8_t* end);
-            static void _encodeIdenticalWords(uint8_t **current, uint8_t* end, std::ostringstream *output);
+            static void _encodeIdenticalWords(const uint8_t **current, const uint8_t* end, std::ostringstream *output);
-            static uint16_t _countIdenticalBytes(uint8_t *start, uint8_t *end);
+            static uint16_t _countIdenticalBytes(const uint8_t *start, const uint8_t *end);
             static bool _hasData(std::istringstream *stream);
             static cpputils::Data _extractData(std::ostringstream *stream);
             static void _parseData(const uint8_t *data, size_t size, std::istringstream *result);

src/blockstore/implementations/encrypted/EncryptedBlock.cpp

@@ -1 +0,0 @@
-#include "EncryptedBlock.h"

src/blockstore/implementations/encrypted/EncryptedBlock.h

@@ -1,219 +0,0 @@
-#pragma once
-#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
-#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
-
-#include "cpp-utils/crypto/cryptopp_byte.h"
-
-#include "../../interface/Block.h"
-#include <cpp-utils/data/Data.h>
-#include "../../interface/BlockStore.h"
-
-#include <cpp-utils/macros.h>
-#include <memory>
-#include <iostream>
-#include <boost/optional.hpp>
-#include <cpp-utils/crypto/symmetric/Cipher.h>
-#include <cpp-utils/assert/assert.h>
-#include <cpp-utils/data/DataUtils.h>
-#include <mutex>
-#include <cpp-utils/logging/logging.h>
-
-namespace blockstore {
-namespace encrypted {
-template<class Cipher> class EncryptedBlockStore;
-
-//TODO Test EncryptedBlock
-
-//TODO Fix mutexes & locks (basically true for all blockstores)
-
-template<class Cipher>
-class EncryptedBlock final: public Block {
-public:
-  BOOST_CONCEPT_ASSERT((cpputils::CipherConcept<Cipher>));
-  static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey);
-  static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key);
-
-  static uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize);
-
-  //TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
-  EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key, cpputils::Data plaintextWithHeader);
-  ~EncryptedBlock();
-
-  const void *data() const override;
-  void write(const void *source, uint64_t offset, uint64_t count) override;
-  void flush() override;
-
-  size_t size() const override;
-  void resize(size_t newSize) override;
-
-  cpputils::unique_ref<Block> releaseBlock();
-
-private:
-  cpputils::unique_ref<Block> _baseBlock; // TODO Do I need the ciphertext block in memory or is the key enough?
-  cpputils::Data _plaintextWithHeader;
-  typename Cipher::EncryptionKey _encKey;
-  bool _dataChanged;
-
-  static constexpr unsigned int HEADER_LENGTH = Key::BINARY_LENGTH;
-
-  void _encryptToBaseBlock();
-  static cpputils::Data _prependKeyHeaderToData(const Key &key, cpputils::Data data);
-  static bool _keyHeaderIsCorrect(const Key &key, const cpputils::Data &data);
-  static cpputils::Data _prependFormatHeader(const cpputils::Data &data);
-  static void _checkFormatHeader(const void *data);
-
-  // This header is prepended to blocks to allow future versions to have compatibility.
-  static constexpr uint16_t FORMAT_VERSION_HEADER = 0;
-
-  std::mutex _mutex;
-
-  DISALLOW_COPY_AND_ASSIGN(EncryptedBlock);
-};
-
-template<class Cipher>
-constexpr unsigned int EncryptedBlock<Cipher>::HEADER_LENGTH;
-
-template<class Cipher>
-constexpr uint16_t EncryptedBlock<Cipher>::FORMAT_VERSION_HEADER;
-
-
-template<class Cipher>
-boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey) {
-  //TODO Is it possible to avoid copying the whole plaintext data into plaintextWithHeader? Maybe an encrypt() object that has an .addData() function and concatenates all data for encryption? Maybe Crypto++ offers this functionality already.
-  cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, std::move(data));
-  cpputils::Data encrypted = Cipher::encrypt((CryptoPP::byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), encKey);
-  //TODO Avoid copying the whole encrypted block into a encryptedWithFormatHeader by creating a Data object with full size and then giving it as an encryption target to Cipher::encrypt()
-  cpputils::Data encryptedWithFormatHeader = _prependFormatHeader(std::move(encrypted));
-  auto baseBlock = baseBlockStore->tryCreate(key, std::move(encryptedWithFormatHeader));
-  if (baseBlock == boost::none) {
-    //TODO Test this code branch
-    return boost::none;
-  }
-
-  return cpputils::make_unique_ref<EncryptedBlock>(std::move(*baseBlock), encKey, std::move(plaintextWithHeader));
-}
-
-template<class Cipher>
-cpputils::Data EncryptedBlock<Cipher>::_prependFormatHeader(const cpputils::Data &data) {
-  cpputils::Data dataWithHeader(sizeof(FORMAT_VERSION_HEADER) + data.size());
-  std::memcpy(dataWithHeader.dataOffset(0), &FORMAT_VERSION_HEADER, sizeof(FORMAT_VERSION_HEADER));
-  std::memcpy(dataWithHeader.dataOffset(sizeof(FORMAT_VERSION_HEADER)), data.data(), data.size());
-  return dataWithHeader;
-}
-
-template<class Cipher>
-boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey) {
-  _checkFormatHeader(baseBlock->data());
-  boost::optional<cpputils::Data> plaintextWithHeader = Cipher::decrypt((CryptoPP::byte*)baseBlock->data() + sizeof(FORMAT_VERSION_HEADER), baseBlock->size() - sizeof(FORMAT_VERSION_HEADER), encKey);
-  if(plaintextWithHeader == boost::none) {
-    //Decryption failed (e.g. an authenticated cipher detected modifications to the ciphertext)
-    cpputils::logging::LOG(cpputils::logging::WARN, "Decrypting block {} failed. Was the block modified by an attacker?", baseBlock->key().ToString());
-    return boost::none;
-  }
-  if(!_keyHeaderIsCorrect(baseBlock->key(), *plaintextWithHeader)) {
-    //The stored key in the block data is incorrect - an attacker might have exchanged the contents with the encrypted data from a different block
-    cpputils::logging::LOG(cpputils::logging::WARN, "Decrypting block {} failed due to invalid block key. Was the block modified by an attacker?", baseBlock->key().ToString());
-    return boost::none;
-  }
-  return cpputils::make_unique_ref<EncryptedBlock<Cipher>>(std::move(baseBlock), encKey, std::move(*plaintextWithHeader));
-}
-
-template<class Cipher>
-void EncryptedBlock<Cipher>::_checkFormatHeader(const void *data) {
-  if (*reinterpret_cast<decltype(FORMAT_VERSION_HEADER)*>(data) != FORMAT_VERSION_HEADER) {
-    throw std::runtime_error("The encrypted block has the wrong format. Was it created with a newer version of CryFS?");
-  }
-}
-
-template<class Cipher>
-cpputils::Data EncryptedBlock<Cipher>::_prependKeyHeaderToData(const Key &key, cpputils::Data data) {
-  static_assert(HEADER_LENGTH >= Key::BINARY_LENGTH, "Key doesn't fit into the header");
-  cpputils::Data result(data.size() + HEADER_LENGTH);
-  std::memcpy(result.data(), key.data(), Key::BINARY_LENGTH);
-  std::memcpy((uint8_t*)result.data() + Key::BINARY_LENGTH, data.data(), data.size());
-  return result;
-}
-
-template<class Cipher>
-bool EncryptedBlock<Cipher>::_keyHeaderIsCorrect(const Key &key, const cpputils::Data &data) {
-  return 0 == std::memcmp(key.data(), data.data(), Key::BINARY_LENGTH);
-}
-
-template<class Cipher>
-EncryptedBlock<Cipher>::EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey, cpputils::Data plaintextWithHeader)
-    :Block(baseBlock->key()),
-   _baseBlock(std::move(baseBlock)),
-   _plaintextWithHeader(std::move(plaintextWithHeader)),
-   _encKey(encKey),
-   _dataChanged(false),
-   _mutex() {
-}
-
-template<class Cipher>
-EncryptedBlock<Cipher>::~EncryptedBlock() {
-  std::unique_lock<std::mutex> lock(_mutex);
-  _encryptToBaseBlock();
-}
-
-template<class Cipher>
-const void *EncryptedBlock<Cipher>::data() const {
-  return (uint8_t*)_plaintextWithHeader.data() + HEADER_LENGTH;
-}
-
-template<class Cipher>
-void EncryptedBlock<Cipher>::write(const void *source, uint64_t offset, uint64_t count) {
-  ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition
-  std::memcpy((uint8_t*)_plaintextWithHeader.data()+HEADER_LENGTH+offset, source, count);
-  _dataChanged = true;
-}
-
-template<class Cipher>
-void EncryptedBlock<Cipher>::flush() {
-  std::unique_lock<std::mutex> lock(_mutex);
-  _encryptToBaseBlock();
-  return _baseBlock->flush();
-}
-
-template<class Cipher>
-size_t EncryptedBlock<Cipher>::size() const {
-  return _plaintextWithHeader.size() - HEADER_LENGTH;
-}
-
-template<class Cipher>
-void EncryptedBlock<Cipher>::resize(size_t newSize) {
-  _plaintextWithHeader = cpputils::DataUtils::resize(std::move(_plaintextWithHeader), newSize + HEADER_LENGTH);
-  _dataChanged = true;
-}
-
-template<class Cipher>
-void EncryptedBlock<Cipher>::_encryptToBaseBlock() {
-  if (_dataChanged) {
-    cpputils::Data encrypted = Cipher::encrypt((CryptoPP::byte*)_plaintextWithHeader.data(), _plaintextWithHeader.size(), _encKey);
-    if (_baseBlock->size() != sizeof(FORMAT_VERSION_HEADER) + encrypted.size()) {
-      _baseBlock->resize(sizeof(FORMAT_VERSION_HEADER) + encrypted.size());
-    }
-    _baseBlock->write(&FORMAT_VERSION_HEADER, 0, sizeof(FORMAT_VERSION_HEADER));
-    _baseBlock->write(encrypted.data(), sizeof(FORMAT_VERSION_HEADER), encrypted.size());
-    _dataChanged = false;
-  }
-}
-
-template<class Cipher>
-cpputils::unique_ref<Block> EncryptedBlock<Cipher>::releaseBlock() {
-  std::unique_lock<std::mutex> lock(_mutex);
-  _encryptToBaseBlock();
-  return std::move(_baseBlock);
-}
-
-template<class Cipher>
-uint64_t EncryptedBlock<Cipher>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) {
-  if (blockSize <= Cipher::ciphertextSize(HEADER_LENGTH) + sizeof(FORMAT_VERSION_HEADER)) {
-    return 0;
-  }
-  return Cipher::plaintextSize(blockSize - sizeof(FORMAT_VERSION_HEADER)) - HEADER_LENGTH;
-}
-
-}
-}
-
-#endif

src/blockstore/implementations/encrypted/EncryptedBlockStore.cpp

@@ -1 +0,0 @@
-#include "EncryptedBlockStore.h"

src/blockstore/implementations/encrypted/EncryptedBlockStore.h

@@ -1,102 +0,0 @@
-#pragma once
-#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
-#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
-
-#include "../../interface/BlockStore.h"
-#include <cpp-utils/macros.h>
-#include <cpp-utils/pointer/cast.h>
-#include "EncryptedBlock.h"
-#include <iostream>
-
-namespace blockstore {
-namespace encrypted {
-
-template<class Cipher>
-class EncryptedBlockStore final: public BlockStore {
-public:
-  EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey);
-
-  //TODO Are createKey() tests included in generic BlockStoreTest? If not, add it!
-  Key createKey() override;
-  boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
-  boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
-  void remove(cpputils::unique_ref<Block> block) override;
-  uint64_t numBlocks() const override;
-  uint64_t estimateNumFreeBytes() const override;
-  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
-
-  //This function should only be used by test cases
-  void __setKey(const typename Cipher::EncryptionKey &encKey);
-
-private:
-  cpputils::unique_ref<BlockStore> _baseBlockStore;
-  typename Cipher::EncryptionKey _encKey;
-
-  DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore);
-};
-
-
-
-template<class Cipher>
-EncryptedBlockStore<Cipher>::EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
- : _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
-}
-
-template<class Cipher>
-Key EncryptedBlockStore<Cipher>::createKey() {
-  return _baseBlockStore->createKey();
-}
-
-template<class Cipher>
-boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::tryCreate(const Key &key, cpputils::Data data) {
-  //TODO Test that this returns boost::none when base blockstore returns nullptr  (for all pass-through-blockstores)
-  //TODO Easier implementation? This is only so complicated because of the cast EncryptedBlock -> Block
-  auto result = EncryptedBlock<Cipher>::TryCreateNew(_baseBlockStore.get(), key, std::move(data), _encKey);
-  if (result == boost::none) {
-    return boost::none;
-  }
-  return cpputils::unique_ref<Block>(std::move(*result));
-}
-
-template<class Cipher>
-boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::load(const Key &key) {
-  auto block = _baseBlockStore->load(key);
-  if (block == boost::none) {
-    //TODO Test this path (for all pass-through-blockstores)
-    return boost::none;
-  }
-  return boost::optional<cpputils::unique_ref<Block>>(EncryptedBlock<Cipher>::TryDecrypt(std::move(*block), _encKey));
-}
-
-template<class Cipher>
-void EncryptedBlockStore<Cipher>::remove(cpputils::unique_ref<Block> block) {
-  auto encryptedBlock = cpputils::dynamic_pointer_move<EncryptedBlock<Cipher>>(block);
-  ASSERT(encryptedBlock != boost::none, "Block is not an EncryptedBlock");
-  auto baseBlock = (*encryptedBlock)->releaseBlock();
-  return _baseBlockStore->remove(std::move(baseBlock));
-}
-
-template<class Cipher>
-uint64_t EncryptedBlockStore<Cipher>::numBlocks() const {
-  return _baseBlockStore->numBlocks();
-}
-
-template<class Cipher>
-uint64_t EncryptedBlockStore<Cipher>::estimateNumFreeBytes() const {
-  return _baseBlockStore->estimateNumFreeBytes();
-}
-
-template<class Cipher>
-void EncryptedBlockStore<Cipher>::__setKey(const typename Cipher::EncryptionKey &encKey) {
-  _encKey = encKey;
-}
-
-template<class Cipher>
-uint64_t EncryptedBlockStore<Cipher>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
-  return EncryptedBlock<Cipher>::blockSizeFromPhysicalBlockSize(_baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize));
-}
-
-}
-}
-
-#endif

src/blockstore/implementations/encrypted/EncryptedBlockStore2.cpp

@@ -0,0 +1 @@
+#include "EncryptedBlockStore2.h"

src/blockstore/implementations/encrypted/EncryptedBlockStore2.h

@@ -0,0 +1,199 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE2_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE2_H_
+
+#include "../../interface/BlockStore2.h"
+#include "cpp-utils/crypto/cryptopp_byte.h"
+#include <cpp-utils/macros.h>
+#include <cpp-utils/crypto/symmetric/Cipher.h>
+#include <cpp-utils/data/SerializationHelper.h>
+
+namespace blockstore {
+namespace encrypted {
+
+//TODO Format version headers
+
+template<class Cipher>
+class EncryptedBlockStore2 final: public BlockStore2 {
+public:
+  BOOST_CONCEPT_ASSERT((cpputils::CipherConcept<Cipher>));
+
+  EncryptedBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const typename Cipher::EncryptionKey &encKey);
+
+  bool tryCreate(const BlockId &blockId, const cpputils::Data &data) override;
+  bool remove(const BlockId &blockId) override;
+  boost::optional<cpputils::Data> load(const BlockId &blockId) const override;
+  void store(const BlockId &blockId, const cpputils::Data &data) override;
+  uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
+  void forEachBlock(std::function<void (const BlockId &)> callback) const override;
+
+  //This function should only be used by test cases
+  void __setKey(const typename Cipher::EncryptionKey &encKey);
+
+private:
+
+  // This header is prepended to blocks to allow future versions to have compatibility.
+#ifndef CRYFS_NO_COMPATIBILITY
+  static constexpr uint16_t FORMAT_VERSION_HEADER_OLD = 0;
+#endif
+  static constexpr uint16_t FORMAT_VERSION_HEADER = 1;
+
+  cpputils::Data _encrypt(const cpputils::Data &data) const;
+  boost::optional<cpputils::Data> _tryDecrypt(const BlockId &blockId, const cpputils::Data &data) const;
+
+  static cpputils::Data _prependFormatHeaderToData(const cpputils::Data &data);
+#ifndef CRYFS_NO_COMPATIBILITY
+  static bool _blockIdHeaderIsCorrect(const BlockId &blockId, const cpputils::Data &data);
+  static cpputils::Data _migrateBlock(const cpputils::Data &data);
+#endif
+  static void _checkFormatHeader(const cpputils::Data &data);
+  static uint16_t _readFormatHeader(const cpputils::Data &data);
+
+  cpputils::unique_ref<BlockStore2> _baseBlockStore;
+  typename Cipher::EncryptionKey _encKey;
+
+  DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore2);
+};
+
+#ifndef CRYFS_NO_COMPATIBILITY
+template<class Cipher>
+constexpr uint16_t EncryptedBlockStore2<Cipher>::FORMAT_VERSION_HEADER_OLD;
+#endif
+
+template<class Cipher>
+constexpr uint16_t EncryptedBlockStore2<Cipher>::FORMAT_VERSION_HEADER;
+
+template<class Cipher>
+inline EncryptedBlockStore2<Cipher>::EncryptedBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
+: _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
+}
+
+template<class Cipher>
+inline bool EncryptedBlockStore2<Cipher>::tryCreate(const BlockId &blockId, const cpputils::Data &data) {
+  cpputils::Data encrypted = _encrypt(data);
+  return _baseBlockStore->tryCreate(blockId, encrypted);
+}
+
+template<class Cipher>
+inline bool EncryptedBlockStore2<Cipher>::remove(const BlockId &blockId) {
+  return _baseBlockStore->remove(blockId);
+}
+
+template<class Cipher>
+inline boost::optional<cpputils::Data> EncryptedBlockStore2<Cipher>::load(const BlockId &blockId) const {
+  auto loaded = _baseBlockStore->load(blockId);
+
+  if (boost::none == loaded) {
+    return boost::optional<cpputils::Data>(boost::none);
+  }
+  return _tryDecrypt(blockId, *loaded);
+}
+
+template<class Cipher>
+inline void EncryptedBlockStore2<Cipher>::store(const BlockId &blockId, const cpputils::Data &data) {
+  cpputils::Data encrypted = _encrypt(data);
+  return _baseBlockStore->store(blockId, encrypted);
+}
+
+template<class Cipher>
+inline uint64_t EncryptedBlockStore2<Cipher>::numBlocks() const {
+  return _baseBlockStore->numBlocks();
+}
+
+template<class Cipher>
+inline uint64_t EncryptedBlockStore2<Cipher>::estimateNumFreeBytes() const {
+  return _baseBlockStore->estimateNumFreeBytes();
+}
+
+template<class Cipher>
+inline uint64_t EncryptedBlockStore2<Cipher>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  uint64_t baseBlockSize = _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
+  if (baseBlockSize <= Cipher::ciphertextSize(0) + sizeof(FORMAT_VERSION_HEADER)) {
+    return 0;
+  }
+  return Cipher::plaintextSize(baseBlockSize - sizeof(FORMAT_VERSION_HEADER));
+}
+
+template<class Cipher>
+inline void EncryptedBlockStore2<Cipher>::forEachBlock(std::function<void (const BlockId &)> callback) const {
+  return _baseBlockStore->forEachBlock(std::move(callback));
+}
+
+template<class Cipher>
+inline cpputils::Data EncryptedBlockStore2<Cipher>::_encrypt(const cpputils::Data &data) const {
+  cpputils::Data encrypted = Cipher::encrypt(static_cast<const CryptoPP::byte*>(data.data()), data.size(), _encKey);
+  return _prependFormatHeaderToData(encrypted);
+}
+
+template<class Cipher>
+inline boost::optional<cpputils::Data> EncryptedBlockStore2<Cipher>::_tryDecrypt(const BlockId &blockId, const cpputils::Data &data) const {
+  _checkFormatHeader(data);
+  boost::optional<cpputils::Data> decrypted = Cipher::decrypt(static_cast<const CryptoPP::byte*>(data.dataOffset(sizeof(FORMAT_VERSION_HEADER))), data.size() - sizeof(FORMAT_VERSION_HEADER), _encKey);
+  if (decrypted == boost::none) {
+    // TODO Log warning
+    return boost::none;
+  }
+
+#ifndef CRYFS_NO_COMPATIBILITY
+  if (FORMAT_VERSION_HEADER_OLD == _readFormatHeader(data)) {
+    if (!_blockIdHeaderIsCorrect(blockId, *decrypted)) {
+      return boost::none;
+    }
+    *decrypted = _migrateBlock(*decrypted);
+    // no need to write migrated back to block store because
+    // this migration happens in line with a migration in IntegrityBlockStore2
+    // which then writes it back
+  }
+#endif
+  return decrypted;
+}
+
+#ifndef CRYFS_NO_COMPATIBILITY
+template<class Cipher>
+inline cpputils::Data EncryptedBlockStore2<Cipher>::_migrateBlock(const cpputils::Data &data) {
+  return data.copyAndRemovePrefix(BlockId::BINARY_LENGTH);
+}
+
+template<class Cipher>
+inline bool EncryptedBlockStore2<Cipher>::_blockIdHeaderIsCorrect(const BlockId &blockId, const cpputils::Data &data) {
+  return blockId == BlockId::FromBinary(data.data());
+}
+#endif
+
+template<class Cipher>
+inline cpputils::Data EncryptedBlockStore2<Cipher>::_prependFormatHeaderToData(const cpputils::Data &data) {
+  cpputils::Data dataWithHeader(sizeof(FORMAT_VERSION_HEADER) + data.size());
+  cpputils::serialize<uint16_t>(dataWithHeader.dataOffset(0), FORMAT_VERSION_HEADER);
+  std::memcpy(dataWithHeader.dataOffset(sizeof(FORMAT_VERSION_HEADER)), data.data(), data.size());
+  return dataWithHeader;
+}
+
+template<class Cipher>
+inline void EncryptedBlockStore2<Cipher>::_checkFormatHeader(const cpputils::Data &data) {
+  const uint16_t formatVersionHeader = _readFormatHeader(data);
+#ifndef CRYFS_NO_COMPATIBILITY
+  const bool formatVersionHeaderValid = formatVersionHeader == FORMAT_VERSION_HEADER || formatVersionHeader == FORMAT_VERSION_HEADER_OLD;
+#else
+  const bool formatVersionHeaderValid = formatVersionHeader == FORMAT_VERSION_HEADER;
+#endif
+  if (!formatVersionHeaderValid) {
+    throw std::runtime_error("The encrypted block has the wrong format. Was it created with a newer version of CryFS?");
+  }
+}
+
+template<class Cipher>
+uint16_t EncryptedBlockStore2<Cipher>::_readFormatHeader(const cpputils::Data &data) {
+  return cpputils::deserialize<uint16_t>(data.data());
+}
+
+template<class Cipher>
+void EncryptedBlockStore2<Cipher>::__setKey(const typename Cipher::EncryptionKey &encKey) {
+  _encKey = encKey;
+}
+
+}
+}
+
+#endif

src/blockstore/implementations/inmemory/InMemoryBlock.cpp

@@ -1,50 +0,0 @@
-#include "InMemoryBlock.h"
-#include "InMemoryBlockStore.h"
-#include <cstring>
-#include <cpp-utils/data/DataUtils.h>
-#include <cpp-utils/assert/assert.h>
-
-using std::make_shared;
-using std::istream;
-using std::ostream;
-using std::ifstream;
-using std::ofstream;
-using std::ios;
-using cpputils::Data;
-
-namespace blockstore {
-namespace inmemory {
-
-InMemoryBlock::InMemoryBlock(const Key &key, Data data)
- : Block(key), _data(make_shared<Data>(std::move(data))) {
-}
-
-InMemoryBlock::InMemoryBlock(const InMemoryBlock &rhs)
- : Block(rhs), _data(rhs._data) {
-}
-
-InMemoryBlock::~InMemoryBlock() {
-}
-
-const void *InMemoryBlock::data() const {
-  return _data->data();
-}
-
-void InMemoryBlock::write(const void *source, uint64_t offset, uint64_t size) {
-  ASSERT(offset <= _data->size() && offset + size <= _data->size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
-  std::memcpy((uint8_t*)_data->data()+offset, source, size);
-}
-
-size_t InMemoryBlock::size() const {
-  return _data->size();
-}
-
-void InMemoryBlock::resize(size_t newSize) {
-    *_data = cpputils::DataUtils::resize(std::move(*_data), newSize);
-}
-
-void InMemoryBlock::flush() {
-}
-
-}
-}

src/blockstore/implementations/inmemory/InMemoryBlock.h

@@ -1,33 +0,0 @@
-#pragma once
-#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
-#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
-
-#include "../../interface/Block.h"
-#include <cpp-utils/data/Data.h>
-
-namespace blockstore {
-namespace inmemory {
-class InMemoryBlockStore;
-
-class InMemoryBlock final: public Block {
-public:
-  InMemoryBlock(const Key &key, cpputils::Data size);
-  InMemoryBlock(const InMemoryBlock &rhs);
-  ~InMemoryBlock();
-
-  const void *data() const override;
-  void write(const void *source, uint64_t offset, uint64_t size) override;
-
-  void flush() override;
-
-  size_t size() const override;
-  void resize(size_t newSize) override;
-
-private:
-  std::shared_ptr<cpputils::Data> _data;
-};
-
-}
-}
-
-#endif

src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp

@@ -1,65 +0,0 @@
-#include "InMemoryBlock.h"
-#include "InMemoryBlockStore.h"
-#include <memory>
-#include <cpp-utils/assert/assert.h>
-#include <cpp-utils/system/get_total_memory.h>
-
-using std::make_unique;
-using std::string;
-using std::mutex;
-using std::lock_guard;
-using std::piecewise_construct;
-using std::make_tuple;
-using cpputils::Data;
-using cpputils::unique_ref;
-using cpputils::make_unique_ref;
-using boost::optional;
-using boost::none;
-
-namespace blockstore {
-namespace inmemory {
-
-InMemoryBlockStore::InMemoryBlockStore()
- : _blocks() {}
-
-optional<unique_ref<Block>> InMemoryBlockStore::tryCreate(const Key &key, Data data) {
-  auto insert_result = _blocks.emplace(piecewise_construct, make_tuple(key), make_tuple(key, std::move(data)));
-
-  if (!insert_result.second) {
-    return none;
-  }
-
-  //Return a pointer to the stored InMemoryBlock
-  return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(insert_result.first->second));
-}
-
-optional<unique_ref<Block>> InMemoryBlockStore::load(const Key &key) {
-  //Return a pointer to the stored InMemoryBlock
-  try {
-    return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(_blocks.at(key)));
-  } catch (const std::out_of_range &e) {
-    return none;
-  }
-}
-
-void InMemoryBlockStore::remove(unique_ref<Block> block) {
-  Key key = block->key();
-  cpputils::destruct(std::move(block));
-  int numRemoved = _blocks.erase(key);
-  ASSERT(1==numRemoved, "Didn't find block to remove");
-}
-
-uint64_t InMemoryBlockStore::numBlocks() const {
-  return _blocks.size();
-}
-
-uint64_t InMemoryBlockStore::estimateNumFreeBytes() const {
-  return cpputils::system::get_total_memory();
-}
-
-uint64_t InMemoryBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
-  return blockSize;
-}
-
-}
-}

src/blockstore/implementations/inmemory/InMemoryBlockStore.h

@@ -1,35 +0,0 @@
-#pragma once
-#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
-#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
-
-#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
-#include <cpp-utils/macros.h>
-
-#include <mutex>
-#include <unordered_map>
-#include "InMemoryBlock.h"
-
-namespace blockstore {
-namespace inmemory {
-
-class InMemoryBlockStore final: public BlockStoreWithRandomKeys {
-public:
-  InMemoryBlockStore();
-
-  boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
-  boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
-  void remove(cpputils::unique_ref<Block> block) override;
-  uint64_t numBlocks() const override;
-  uint64_t estimateNumFreeBytes() const override;
-  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
-
-private:
-  std::unordered_map<Key, InMemoryBlock> _blocks;
-
-  DISALLOW_COPY_AND_ASSIGN(InMemoryBlockStore);
-};
-
-}
-}
-
-#endif

src/blockstore/implementations/inmemory/InMemoryBlockStore2.cpp

@@ -0,0 +1,96 @@
+#include "InMemoryBlockStore2.h"
+#include <memory>
+#include <cpp-utils/assert/assert.h>
+#include <cpp-utils/system/get_total_memory.h>
+
+using std::string;
+using std::mutex;
+using std::make_pair;
+using std::vector;
+using cpputils::Data;
+using boost::optional;
+using boost::none;
+
+namespace blockstore {
+namespace inmemory {
+
+InMemoryBlockStore2::InMemoryBlockStore2()
+ : _blocks() {}
+
+bool InMemoryBlockStore2::tryCreate(const BlockId &blockId, const Data &data) {
+  std::unique_lock<std::mutex> lock(_mutex);
+  return _tryCreate(blockId, data);
+}
+
+bool InMemoryBlockStore2::_tryCreate(const BlockId &blockId, const Data &data) {
+  auto result = _blocks.insert(make_pair(blockId, data.copy()));
+  return result.second; // Return if insertion was successful (i.e. blockId didn't exist yet)
+}
+
+bool InMemoryBlockStore2::remove(const BlockId &blockId) {
+  std::unique_lock<std::mutex> lock(_mutex);
+  auto found = _blocks.find(blockId);
+  if (found == _blocks.end()) {
+    // BlockId not found
+    return false;
+  }
+
+  _blocks.erase(found);
+  return true;
+}
+
+optional<Data> InMemoryBlockStore2::load(const BlockId &blockId) const {
+  std::unique_lock<std::mutex> lock(_mutex);
+  auto found = _blocks.find(blockId);
+  if (found == _blocks.end()) {
+    return boost::none;
+  }
+  return found->second.copy();
+}
+
+void InMemoryBlockStore2::store(const BlockId &blockId, const Data &data) {
+  std::unique_lock<std::mutex> lock(_mutex);
+  auto found = _blocks.find(blockId);
+  if (found == _blocks.end()) {
+    bool success = _tryCreate(blockId, data);
+    if (!success) {
+      throw std::runtime_error("Could neither save nor create the block in InMemoryBlockStore::store()");
+    }
+  } else {
+    // TODO Would have better performance: found->second.overwriteWith(data)
+    found->second = data.copy();
+  }
+}
+
+uint64_t InMemoryBlockStore2::numBlocks() const {
+  std::unique_lock<std::mutex> lock(_mutex);
+  return _blocks.size();
+}
+
+uint64_t InMemoryBlockStore2::estimateNumFreeBytes() const {
+  return cpputils::system::get_total_memory();
+}
+
+uint64_t InMemoryBlockStore2::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  return blockSize;
+}
+
+vector<BlockId> InMemoryBlockStore2::_allBlockIds() const {
+  std::unique_lock<std::mutex> lock(_mutex);
+  vector<BlockId> result;
+  result.reserve(_blocks.size());
+  for (const auto &entry : _blocks) {
+    result.push_back(entry.first);
+  }
+  return result;
+}
+
+void InMemoryBlockStore2::forEachBlock(std::function<void (const BlockId &)> callback) const {
+  auto blockIds = _allBlockIds();
+  for (const auto &blockId : blockIds) {
+    callback(blockId);
+  }
+}
+
+}
+}

src/blockstore/implementations/inmemory/InMemoryBlockStore2.h

@@ -0,0 +1,38 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE2_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE2_H_
+
+#include "../../interface/BlockStore2.h"
+#include <cpp-utils/macros.h>
+#include <unordered_map>
+
+namespace blockstore {
+namespace inmemory {
+
+class InMemoryBlockStore2 final: public BlockStore2 {
+public:
+  InMemoryBlockStore2();
+
+  bool tryCreate(const BlockId &blockId, const cpputils::Data &data) override;
+  bool remove(const BlockId &blockId) override;
+  boost::optional<cpputils::Data> load(const BlockId &blockId) const override;
+  void store(const BlockId &blockId, const cpputils::Data &data) override;
+  uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
+  void forEachBlock(std::function<void (const BlockId &)> callback) const override;
+
+private:
+  std::vector<BlockId> _allBlockIds() const;
+  bool _tryCreate(const BlockId &blockId, const cpputils::Data &data);
+
+  std::unordered_map<BlockId, cpputils::Data> _blocks;
+  mutable std::mutex _mutex;
+
+  DISALLOW_COPY_AND_ASSIGN(InMemoryBlockStore2);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/integrity/ClientIdAndBlockId.cpp

@@ -0,0 +1 @@
+#include "ClientIdAndBlockId.h"

src/blockstore/implementations/integrity/ClientIdAndBlockId.h

@@ -0,0 +1,36 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_CLIENTIDANDBLOCKID_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_CLIENTIDANDBLOCKID_H_
+
+#include <utility>
+#include "blockstore/utils/BlockId.h"
+
+namespace blockstore {
+    namespace integrity {
+
+        struct ClientIdAndBlockId final {
+            ClientIdAndBlockId(uint32_t clientId_, BlockId blockId_): clientId(clientId_), blockId(blockId_) {}
+
+            uint32_t clientId;
+            BlockId blockId;
+        };
+
+    }
+}
+
+// Allow using it in std::unordered_set / std::unordered_map
+namespace std {
+    template<> struct hash<blockstore::integrity::ClientIdAndBlockId> {
+        size_t operator()(const blockstore::integrity::ClientIdAndBlockId &ref) const {
+            return std::hash<uint32_t>()(ref.clientId) ^ std::hash<blockstore::BlockId>()(ref.blockId);
+        }
+    };
+
+    template<> struct equal_to<blockstore::integrity::ClientIdAndBlockId> {
+        size_t operator()(const blockstore::integrity::ClientIdAndBlockId &lhs, const blockstore::integrity::ClientIdAndBlockId &rhs) const {
+            return lhs.clientId == rhs.clientId && lhs.blockId == rhs.blockId;
+        }
+    };
+}
+
+#endif

src/blockstore/implementations/integrity/IntegrityBlockStore2.cpp

@@ -0,0 +1,223 @@
+#include <blockstore/interface/BlockStore2.h>
+#include "IntegrityBlockStore2.h"
+#include "KnownBlockVersions.h"
+#include <cpp-utils/data/SerializationHelper.h>
+
+using cpputils::Data;
+using cpputils::unique_ref;
+using cpputils::serialize;
+using cpputils::deserialize;
+using std::string;
+using boost::optional;
+using boost::none;
+using namespace cpputils::logging;
+
+namespace blockstore {
+namespace integrity {
+
+#ifndef CRYFS_NO_COMPATIBILITY
+constexpr uint16_t IntegrityBlockStore2::FORMAT_VERSION_HEADER_OLD;
+#endif
+constexpr uint16_t IntegrityBlockStore2::FORMAT_VERSION_HEADER;
+constexpr uint64_t IntegrityBlockStore2::VERSION_ZERO;
+constexpr unsigned int IntegrityBlockStore2::ID_HEADER_OFFSET;
+constexpr unsigned int IntegrityBlockStore2::CLIENTID_HEADER_OFFSET;
+constexpr unsigned int IntegrityBlockStore2::VERSION_HEADER_OFFSET;
+constexpr unsigned int IntegrityBlockStore2::HEADER_LENGTH;
+
+Data IntegrityBlockStore2::_prependHeaderToData(const BlockId& blockId, uint32_t myClientId, uint64_t version, const Data &data) {
+  static_assert(HEADER_LENGTH == sizeof(FORMAT_VERSION_HEADER) + BlockId::BINARY_LENGTH + sizeof(myClientId) + sizeof(version), "Wrong header length");
+  Data result(data.size() + HEADER_LENGTH);
+  serialize<uint16_t>(result.dataOffset(0), FORMAT_VERSION_HEADER);
+  blockId.ToBinary(result.dataOffset(ID_HEADER_OFFSET));
+  serialize<uint32_t>(result.dataOffset(CLIENTID_HEADER_OFFSET), myClientId);
+  serialize<uint64_t>(result.dataOffset(VERSION_HEADER_OFFSET), version);
+  std::memcpy(result.dataOffset(HEADER_LENGTH), data.data(), data.size());
+  return result;
+}
+
+void IntegrityBlockStore2::_checkHeader(const BlockId &blockId, const Data &data) const {
+  _checkFormatHeader(data);
+  _checkIdHeader(blockId, data);
+  _checkVersionHeader(blockId, data);
+}
+
+void IntegrityBlockStore2::_checkFormatHeader(const Data &data) const {
+  if (FORMAT_VERSION_HEADER != _readFormatHeader(data)) {
+    throw std::runtime_error("The versioned block has the wrong format. Was it created with a newer version of CryFS?");
+  }
+}
+
+void IntegrityBlockStore2::_checkVersionHeader(const BlockId &blockId, const Data &data) const {
+  uint32_t clientId = _readClientId(data);
+  uint64_t version = _readVersion(data);
+
+  if(!_knownBlockVersions.checkAndUpdateVersion(clientId, blockId, version)) {
+    integrityViolationDetected("The block version number is too low. Did an attacker try to roll back the block or to re-introduce a deleted block?");
+  }
+}
+
+void IntegrityBlockStore2::_checkIdHeader(const BlockId &expectedBlockId, const Data &data) const {
+  BlockId actualBlockId = _readBlockId(data);
+  if (expectedBlockId != actualBlockId) {
+    integrityViolationDetected("The block id is wrong. Did an attacker try to rename some blocks?");
+  }
+}
+
+uint16_t IntegrityBlockStore2::_readFormatHeader(const Data &data) {
+  return deserialize<uint16_t>(data.data());
+}
+
+uint32_t IntegrityBlockStore2::_readClientId(const Data &data) {
+  return deserialize<uint32_t>(data.dataOffset(CLIENTID_HEADER_OFFSET));
+}
+
+BlockId IntegrityBlockStore2::_readBlockId(const Data &data) {
+  return BlockId::FromBinary(data.dataOffset(ID_HEADER_OFFSET));
+}
+
+uint64_t IntegrityBlockStore2::_readVersion(const Data &data) {
+  return deserialize<uint64_t>(data.dataOffset(VERSION_HEADER_OFFSET));
+}
+
+Data IntegrityBlockStore2::_removeHeader(const Data &data) {
+  return data.copyAndRemovePrefix(HEADER_LENGTH);
+}
+
+void IntegrityBlockStore2::_checkNoPastIntegrityViolations() const {
+  if (_integrityViolationDetected) {
+    throw std::runtime_error(string() +
+                             "There was an integrity violation detected. Preventing any further access to the file system. " +
+                             "If you want to reset the integrity data (i.e. accept changes made by a potential attacker), " +
+                             "please unmount the file system and delete the following file before re-mounting it: " +
+                             _knownBlockVersions.path().string());
+  }
+}
+
+void IntegrityBlockStore2::integrityViolationDetected(const string &reason) const {
+  if (_allowIntegrityViolations) {
+    LOG(WARN, "Integrity violation (but integrity checks are disabled): {}", reason);
+    return;
+  }
+  _integrityViolationDetected = true;
+  throw IntegrityViolationError(reason);
+}
+
+IntegrityBlockStore2::IntegrityBlockStore2(unique_ref<BlockStore2> baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId, bool allowIntegrityViolations, bool missingBlockIsIntegrityViolation)
+: _baseBlockStore(std::move(baseBlockStore)), _knownBlockVersions(integrityFilePath, myClientId), _allowIntegrityViolations(allowIntegrityViolations), _missingBlockIsIntegrityViolation(missingBlockIsIntegrityViolation), _integrityViolationDetected(false) {
+}
+
+bool IntegrityBlockStore2::tryCreate(const BlockId &blockId, const Data &data) {
+  _checkNoPastIntegrityViolations();
+  uint64_t version = _knownBlockVersions.incrementVersion(blockId);
+  Data dataWithHeader = _prependHeaderToData(blockId, _knownBlockVersions.myClientId(), version, data);
+  return _baseBlockStore->tryCreate(blockId, dataWithHeader);
+}
+
+bool IntegrityBlockStore2::remove(const BlockId &blockId) {
+  _checkNoPastIntegrityViolations();
+  _knownBlockVersions.markBlockAsDeleted(blockId);
+  return _baseBlockStore->remove(blockId);
+}
+
+optional<Data> IntegrityBlockStore2::load(const BlockId &blockId) const {
+  _checkNoPastIntegrityViolations();
+  auto loaded = _baseBlockStore->load(blockId);
+  if (none == loaded) {
+    if (_missingBlockIsIntegrityViolation && _knownBlockVersions.blockShouldExist(blockId)) {
+      integrityViolationDetected("A block that should exist wasn't found. Did an attacker delete it?");
+    }
+    return optional<Data>(none);
+  }
+#ifndef CRYFS_NO_COMPATIBILITY
+  if (FORMAT_VERSION_HEADER_OLD == _readFormatHeader(*loaded)) {
+    Data migrated = _migrateBlock(blockId, *loaded);
+    _checkHeader(blockId, migrated);
+    Data content = _removeHeader(migrated);
+    const_cast<IntegrityBlockStore2*>(this)->store(blockId, content);
+    return optional<Data>(_removeHeader(migrated));
+  }
+#endif
+  _checkHeader(blockId, *loaded);
+  return optional<Data>(_removeHeader(*loaded));
+}
+
+#ifndef CRYFS_NO_COMPATIBILITY
+Data IntegrityBlockStore2::_migrateBlock(const BlockId &blockId, const Data &data) {
+  Data migrated(data.size() + BlockId::BINARY_LENGTH);
+  serialize<uint16_t>(migrated.dataOffset(0), FORMAT_VERSION_HEADER);
+  blockId.ToBinary(migrated.dataOffset(ID_HEADER_OFFSET));
+  std::memcpy(migrated.dataOffset(ID_HEADER_OFFSET + BlockId::BINARY_LENGTH), data.dataOffset(sizeof(FORMAT_VERSION_HEADER)), data.size() - sizeof(FORMAT_VERSION_HEADER));
+  ASSERT(migrated.size() == sizeof(FORMAT_VERSION_HEADER) + BlockId::BINARY_LENGTH + (data.size() - sizeof(FORMAT_VERSION_HEADER)), "Wrong offset computation");
+  return migrated;
+}
+#endif
+
+void IntegrityBlockStore2::store(const BlockId &blockId, const Data &data) {
+  _checkNoPastIntegrityViolations();
+  uint64_t version = _knownBlockVersions.incrementVersion(blockId);
+  Data dataWithHeader = _prependHeaderToData(blockId, _knownBlockVersions.myClientId(), version, data);
+  return _baseBlockStore->store(blockId, dataWithHeader);
+}
+
+uint64_t IntegrityBlockStore2::numBlocks() const {
+  return _baseBlockStore->numBlocks();
+}
+
+uint64_t IntegrityBlockStore2::estimateNumFreeBytes() const {
+  return _baseBlockStore->estimateNumFreeBytes();
+}
+
+uint64_t IntegrityBlockStore2::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  uint64_t baseBlockSize = _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
+  if (baseBlockSize <= HEADER_LENGTH) {
+    return 0;
+  }
+  return baseBlockSize - HEADER_LENGTH;
+}
+
+void IntegrityBlockStore2::forEachBlock(std::function<void (const BlockId &)> callback) const {
+  if (!_missingBlockIsIntegrityViolation) {
+    return _baseBlockStore->forEachBlock(std::move(callback));
+  }
+
+  std::unordered_set<blockstore::BlockId> existingBlocks = _knownBlockVersions.existingBlocks();
+  _baseBlockStore->forEachBlock([&existingBlocks, callback] (const BlockId &blockId) {
+    callback(blockId);
+
+    auto found = existingBlocks.find(blockId);
+    if (found != existingBlocks.end()) {
+      existingBlocks.erase(found);
+    }
+  });
+  if (!existingBlocks.empty()) {
+    integrityViolationDetected("A block that should have existed wasn't found.");
+  }
+}
+
+#ifndef CRYFS_NO_COMPATIBILITY
+void IntegrityBlockStore2::migrateFromBlockstoreWithoutVersionNumbers(BlockStore2 *baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId) {
+  std::cout << "Migrating file system for integrity features. Please don't interrupt this process. This can take a while..." << std::flush;
+  KnownBlockVersions knownBlockVersions(integrityFilePath, myClientId);
+  baseBlockStore->forEachBlock([&baseBlockStore, &knownBlockVersions] (const BlockId &blockId) {
+    migrateBlockFromBlockstoreWithoutVersionNumbers(baseBlockStore, blockId, &knownBlockVersions);
+  });
+  std::cout << "done" << std::endl;
+}
+
+void IntegrityBlockStore2::migrateBlockFromBlockstoreWithoutVersionNumbers(blockstore::BlockStore2* baseBlockStore, const blockstore::BlockId& blockId, KnownBlockVersions *knownBlockVersions) {
+  uint64_t version = knownBlockVersions->incrementVersion(blockId);
+
+  auto data_ = baseBlockStore->load(blockId);
+  if (data_ == boost::none) {
+    LOG(WARN, "Block not found, but was returned from forEachBlock before");
+    return;
+  }
+  cpputils::Data data = std::move(*data_);
+  cpputils::Data dataWithHeader = _prependHeaderToData(blockId, knownBlockVersions->myClientId(), version, data);
+  baseBlockStore->store(blockId, dataWithHeader);
+}
+#endif
+
+}
+}

src/blockstore/implementations/integrity/IntegrityBlockStore2.h

@@ -0,0 +1,79 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_INTEGRITYBLOCKSTORE2_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_INTEGRITYBLOCKSTORE2_H_
+
+#include "../../interface/BlockStore2.h"
+#include <cpp-utils/macros.h>
+#include "KnownBlockVersions.h"
+#include "IntegrityViolationError.h"
+
+namespace blockstore {
+namespace integrity {
+
+//TODO Format version headers
+
+// This blockstore implements integrity measures.
+// It depends on being used on top of an encrypted block store that protects integrity of the block contents (i.e. uses an authenticated cipher).
+class IntegrityBlockStore2 final: public BlockStore2 {
+public:
+  IntegrityBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId, bool allowIntegrityViolations, bool missingBlockIsIntegrityViolation);
+
+  bool tryCreate(const BlockId &blockId, const cpputils::Data &data) override;
+  bool remove(const BlockId &blockId) override;
+  boost::optional<cpputils::Data> load(const BlockId &blockId) const override;
+  void store(const BlockId &blockId, const cpputils::Data &data) override;
+  uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
+  void forEachBlock(std::function<void (const BlockId &)> callback) const override;
+
+private:
+  // This format version is prepended to blocks to allow future versions to have compatibility.
+#ifndef CRYFS_NO_COMPATIBILITY
+  static constexpr uint16_t FORMAT_VERSION_HEADER_OLD = 0;
+#endif
+  static constexpr uint16_t FORMAT_VERSION_HEADER = 1;
+
+public:
+  static constexpr uint64_t VERSION_ZERO = 0;
+  static constexpr unsigned int ID_HEADER_OFFSET = sizeof(FORMAT_VERSION_HEADER);
+  static constexpr unsigned int CLIENTID_HEADER_OFFSET = ID_HEADER_OFFSET + BlockId::BINARY_LENGTH;
+  static constexpr unsigned int VERSION_HEADER_OFFSET = CLIENTID_HEADER_OFFSET + sizeof(uint32_t);
+  static constexpr unsigned int HEADER_LENGTH = VERSION_HEADER_OFFSET + sizeof(VERSION_ZERO);
+
+#ifndef CRYFS_NO_COMPATIBILITY
+  static void migrateFromBlockstoreWithoutVersionNumbers(BlockStore2 *baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId);
+  static void migrateBlockFromBlockstoreWithoutVersionNumbers(BlockStore2* baseBlockStore, const blockstore::BlockId &blockId, KnownBlockVersions *knownBlockVersions);
+#endif
+
+private:
+
+  static cpputils::Data _prependHeaderToData(const BlockId &blockId, uint32_t myClientId, uint64_t version, const cpputils::Data &data);
+  void _checkHeader(const BlockId &blockId, const cpputils::Data &data) const;
+  void _checkFormatHeader(const cpputils::Data &data) const;
+  void _checkIdHeader(const BlockId &expectedBlockId, const cpputils::Data &data) const;
+  void _checkVersionHeader(const BlockId &blockId, const cpputils::Data &data) const;
+  static uint16_t _readFormatHeader(const cpputils::Data &data);
+  static uint32_t _readClientId(const cpputils::Data &data);
+  static BlockId _readBlockId(const cpputils::Data &data);
+  static uint64_t _readVersion(const cpputils::Data &data);
+#ifndef CRYFS_NO_COMPATIBILITY
+  static cpputils::Data _migrateBlock(const BlockId &blockId, const cpputils::Data &data);
+#endif
+  static cpputils::Data _removeHeader(const cpputils::Data &data);
+  void _checkNoPastIntegrityViolations() const;
+  void integrityViolationDetected(const std::string &reason) const;
+
+  cpputils::unique_ref<BlockStore2> _baseBlockStore;
+  mutable KnownBlockVersions _knownBlockVersions;
+  const bool _allowIntegrityViolations;
+  const bool _missingBlockIsIntegrityViolation;
+  mutable bool _integrityViolationDetected;
+
+  DISALLOW_COPY_AND_ASSIGN(IntegrityBlockStore2);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/integrity/IntegrityViolationError.cpp

@@ -0,0 +1 @@
+#include "IntegrityViolationError.h"

src/blockstore/implementations/integrity/IntegrityViolationError.h

@@ -0,0 +1,27 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_INTEGRITYVIOLATIONERROR_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_INTEGRITYVIOLATIONERROR_H_
+
+#include <cpp-utils/macros.h>
+#include <string>
+#include <stdexcept>
+
+namespace blockstore {
+    namespace integrity {
+
+        class IntegrityViolationError final : public std::runtime_error {
+        private:
+            // Constructor is private to make sure that only IntegrityBlockStore can throw this exception.
+            // This is because IntegrityBlockStore wants to know about integrity violations and
+            // block all further file system access if it happens.
+            IntegrityViolationError(const std::string &reason)
+                    : std::runtime_error("Integrity violation: " + reason) {
+            }
+            friend class IntegrityBlockStore2;
+        };
+
+
+    }
+}
+
+#endif

src/blockstore/implementations/integrity/KnownBlockVersions.cpp

@@ -0,0 +1,218 @@
+#include <fstream>
+#include <cpp-utils/random/Random.h>
+#include <unordered_set>
+#include "KnownBlockVersions.h"
+
+namespace bf = boost::filesystem;
+using std::pair;
+using std::string;
+using std::unique_lock;
+using std::mutex;
+using boost::optional;
+using boost::none;
+using cpputils::Data;
+using cpputils::Serializer;
+using cpputils::Deserializer;
+
+namespace blockstore {
+namespace integrity {
+
+const string KnownBlockVersions::HEADER = "cryfs.integritydata.knownblockversions;0";
+constexpr uint32_t KnownBlockVersions::CLIENT_ID_FOR_DELETED_BLOCK;
+
+KnownBlockVersions::KnownBlockVersions(const bf::path &stateFilePath, uint32_t myClientId)
+        :_knownVersions(), _lastUpdateClientId(), _stateFilePath(stateFilePath), _myClientId(myClientId), _mutex(), _valid(true) {
+    unique_lock<mutex> lock(_mutex);
+    ASSERT(_myClientId != CLIENT_ID_FOR_DELETED_BLOCK, "This is not a valid client id");
+    _loadStateFile();
+}
+
+KnownBlockVersions::KnownBlockVersions(KnownBlockVersions &&rhs) // NOLINT (intentionally not noexcept)
+        : _knownVersions(), _lastUpdateClientId(), _stateFilePath(), _myClientId(0), _mutex(), _valid(true) {
+    unique_lock<mutex> rhsLock(rhs._mutex);
+    unique_lock<mutex> lock(_mutex);
+    _knownVersions = std::move(rhs._knownVersions);
+    _lastUpdateClientId = std::move(rhs._lastUpdateClientId);
+    _stateFilePath = std::move(rhs._stateFilePath);
+    _myClientId = rhs._myClientId;
+    rhs._valid = false;
+}
+
+KnownBlockVersions::~KnownBlockVersions() {
+    unique_lock<mutex> lock(_mutex);
+    if (_valid) {
+        _saveStateFile();
+    }
+}
+
+bool KnownBlockVersions::checkAndUpdateVersion(uint32_t clientId, const BlockId &blockId, uint64_t version) {
+    unique_lock<mutex> lock(_mutex);
+    ASSERT(clientId != CLIENT_ID_FOR_DELETED_BLOCK, "This is not a valid client id");
+
+    ASSERT(version > 0, "Version has to be >0"); // Otherwise we wouldn't handle notexisting entries correctly.
+    ASSERT(_valid, "Object not valid due to a std::move");
+
+    uint64_t &found = _knownVersions[{clientId, blockId}]; // If the entry doesn't exist, this creates it with value 0.
+    if (found > version) {
+        // This client already published a newer block version. Rollbacks are not allowed.
+        return false;
+    }
+
+    uint32_t &lastUpdateClientId = _lastUpdateClientId[blockId]; // If entry doesn't exist, this creates it with value 0. However, in this case, found == 0 (and version > 0), which means found != version.
+    if (found == version && lastUpdateClientId != clientId) {
+        // This is a roll back to the "newest" block of client [clientId], which was since then superseded by a version from client _lastUpdateClientId[blockId].
+        // This is not allowed.
+        return false;
+    }
+
+    found = version;
+    lastUpdateClientId = clientId;
+    return true;
+}
+
+uint64_t KnownBlockVersions::incrementVersion(const BlockId &blockId) {
+    unique_lock<mutex> lock(_mutex);
+    uint64_t &found = _knownVersions[{_myClientId, blockId}]; // If the entry doesn't exist, this creates it with value 0.
+    uint64_t newVersion = found + 1;
+    if (newVersion == std::numeric_limits<uint64_t>::max()) {
+        // It's *very* unlikely we ever run out of version numbers in 64bit...but just to be sure...
+        throw std::runtime_error("Version overflow");
+    }
+    found = newVersion;
+    _lastUpdateClientId[blockId] = _myClientId;
+    return found;
+}
+
+void KnownBlockVersions::_loadStateFile() {
+    optional<Data> file = Data::LoadFromFile(_stateFilePath);
+    if (file == none) {
+        // File doesn't exist means we loaded empty state.
+        return;
+    }
+
+    Deserializer deserializer(&*file);
+    if (HEADER != deserializer.readString()) {
+        throw std::runtime_error("Invalid local state: Invalid integrity file header.");
+    }
+    _deserializeKnownVersions(&deserializer);
+    _deserializeLastUpdateClientIds(&deserializer);
+
+    deserializer.finished();
+};
+
+
+void KnownBlockVersions::_saveStateFile() const {
+    Serializer serializer(
+            Serializer::StringSize(HEADER) +
+            sizeof(uint64_t) + _knownVersions.size() * (sizeof(uint32_t) + BlockId::BINARY_LENGTH + sizeof(uint64_t)) +
+            sizeof(uint64_t) + _lastUpdateClientId.size() * (BlockId::BINARY_LENGTH + sizeof(uint32_t)));
+    serializer.writeString(HEADER);
+    _serializeKnownVersions(&serializer);
+    _serializeLastUpdateClientIds(&serializer);
+
+    serializer.finished().StoreToFile(_stateFilePath);
+}
+
+void KnownBlockVersions::_deserializeKnownVersions(Deserializer *deserializer) {
+    uint64_t numEntries = deserializer->readUint64();
+    _knownVersions.clear();
+    _knownVersions.reserve(static_cast<uint64_t>(1.2 * numEntries)); // Reserve for factor 1.2 more, so the file system doesn't immediately have to resize it on the first new block.
+    for (uint64_t i = 0 ; i < numEntries; ++i) {
+        auto entry = _deserializeKnownVersionsEntry(deserializer);
+        _knownVersions.insert(entry);
+    }
+}
+
+void KnownBlockVersions::_serializeKnownVersions(Serializer *serializer) const {
+    uint64_t numEntries = _knownVersions.size();
+    serializer->writeUint64(numEntries);
+
+    for (const auto &entry : _knownVersions) {
+        _serializeKnownVersionsEntry(serializer, entry);
+    }
+}
+
+pair<ClientIdAndBlockId, uint64_t> KnownBlockVersions::_deserializeKnownVersionsEntry(Deserializer *deserializer) {
+    uint32_t clientId = deserializer->readUint32();
+    BlockId blockId(deserializer->readFixedSizeData<BlockId::BINARY_LENGTH>());
+    uint64_t version = deserializer->readUint64();
+
+    return {{clientId, blockId}, version};
+};
+
+void KnownBlockVersions::_serializeKnownVersionsEntry(Serializer *serializer, const pair<ClientIdAndBlockId, uint64_t> &entry) {
+    serializer->writeUint32(entry.first.clientId);
+    serializer->writeFixedSizeData<BlockId::BINARY_LENGTH>(entry.first.blockId.data());
+    serializer->writeUint64(entry.second);
+}
+
+void KnownBlockVersions::_deserializeLastUpdateClientIds(Deserializer *deserializer) {
+    uint64_t numEntries = deserializer->readUint64();
+    _lastUpdateClientId.clear();
+    _lastUpdateClientId.reserve(static_cast<uint64_t>(1.2 * numEntries)); // Reserve for factor 1.2 more, so the file system doesn't immediately have to resize it on the first new block.
+    for (uint64_t i = 0 ; i < numEntries; ++i) {
+        auto entry = _deserializeLastUpdateClientIdEntry(deserializer);
+        _lastUpdateClientId.insert(entry);
+    }
+}
+
+void KnownBlockVersions::_serializeLastUpdateClientIds(Serializer *serializer) const {
+    uint64_t numEntries = _lastUpdateClientId.size();
+    serializer->writeUint64(numEntries);
+
+    for (const auto &entry : _lastUpdateClientId) {
+        _serializeLastUpdateClientIdEntry(serializer, entry);
+    }
+}
+
+pair<BlockId, uint32_t> KnownBlockVersions::_deserializeLastUpdateClientIdEntry(Deserializer *deserializer) {
+    BlockId blockId(deserializer->readFixedSizeData<BlockId::BINARY_LENGTH>());
+    uint32_t clientId = deserializer->readUint32();
+
+    return {blockId, clientId};
+};
+
+void KnownBlockVersions::_serializeLastUpdateClientIdEntry(Serializer *serializer, const pair<BlockId, uint32_t> &entry) {
+    serializer->writeFixedSizeData<BlockId::BINARY_LENGTH>(entry.first.data());
+    serializer->writeUint32(entry.second);
+}
+
+uint32_t KnownBlockVersions::myClientId() const {
+    return _myClientId;
+}
+
+uint64_t KnownBlockVersions::getBlockVersion(uint32_t clientId, const BlockId &blockId) const {
+    unique_lock<mutex> lock(_mutex);
+    return _knownVersions.at({clientId, blockId});
+}
+
+void KnownBlockVersions::markBlockAsDeleted(const BlockId &blockId) {
+    _lastUpdateClientId[blockId] = CLIENT_ID_FOR_DELETED_BLOCK;
+}
+
+bool KnownBlockVersions::blockShouldExist(const BlockId &blockId) const {
+    auto found = _lastUpdateClientId.find(blockId);
+    if (found == _lastUpdateClientId.end()) {
+        // We've never seen (i.e. loaded) this block. So we can't say it has to exist.
+        return false;
+    }
+    // We've seen the block before. If we didn't delete it, it should exist (only works for single-client scenario).
+    return found->second != CLIENT_ID_FOR_DELETED_BLOCK;
+}
+
+std::unordered_set<BlockId> KnownBlockVersions::existingBlocks() const {
+    std::unordered_set<BlockId> result;
+    for (const auto &entry : _lastUpdateClientId) {
+        if (entry.second != CLIENT_ID_FOR_DELETED_BLOCK) {
+            result.insert(entry.first);
+        }
+    }
+    return result;
+}
+
+const bf::path &KnownBlockVersions::path() const {
+    return _stateFilePath;
+}
+
+}
+}

src/blockstore/implementations/integrity/KnownBlockVersions.h

@@ -0,0 +1,74 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_KNOWNBLOCKVERSIONS_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_KNOWNBLOCKVERSIONS_H_
+
+#include <cpp-utils/macros.h>
+#include <blockstore/utils/BlockId.h>
+#include <boost/filesystem/path.hpp>
+#include <boost/optional.hpp>
+#include "ClientIdAndBlockId.h"
+#include <cpp-utils/data/Deserializer.h>
+#include <cpp-utils/data/Serializer.h>
+#include <mutex>
+#include <unordered_set>
+
+namespace blockstore {
+    namespace integrity {
+
+        class KnownBlockVersions final {
+        public:
+            KnownBlockVersions(const boost::filesystem::path &stateFilePath, uint32_t myClientId);
+            KnownBlockVersions(KnownBlockVersions &&rhs); // NOLINT (intentionally not noexcept)
+            ~KnownBlockVersions();
+
+			WARN_UNUSED_RESULT
+            bool checkAndUpdateVersion(uint32_t clientId, const BlockId &blockId, uint64_t version);
+
+            uint64_t incrementVersion(const BlockId &blockId);
+
+            void markBlockAsDeleted(const BlockId &blockId);
+
+            bool blockShouldExist(const BlockId &blockId) const;
+            std::unordered_set<BlockId> existingBlocks() const;
+
+            uint64_t getBlockVersion(uint32_t clientId, const BlockId &blockId) const;
+
+            uint32_t myClientId() const;
+            const boost::filesystem::path &path() const;
+
+            static constexpr uint32_t CLIENT_ID_FOR_DELETED_BLOCK = 0;
+
+        private:
+            std::unordered_map<ClientIdAndBlockId, uint64_t> _knownVersions;
+            std::unordered_map<BlockId, uint32_t> _lastUpdateClientId; // The client who last updated the block
+
+            boost::filesystem::path _stateFilePath;
+            uint32_t _myClientId;
+            mutable std::mutex _mutex;
+            bool _valid;
+
+            static const std::string HEADER;
+
+            void _loadStateFile();
+            void _saveStateFile() const;
+
+            void _deserializeKnownVersions(cpputils::Deserializer *deserializer);
+            void _serializeKnownVersions(cpputils::Serializer *serializer) const;
+
+            static std::pair<ClientIdAndBlockId, uint64_t> _deserializeKnownVersionsEntry(cpputils::Deserializer *deserializer);
+            static void _serializeKnownVersionsEntry(cpputils::Serializer *serializer, const std::pair<ClientIdAndBlockId, uint64_t> &entry);
+
+            void _deserializeLastUpdateClientIds(cpputils::Deserializer *deserializer);
+            void _serializeLastUpdateClientIds(cpputils::Serializer *serializer) const;
+
+            static std::pair<BlockId, uint32_t> _deserializeLastUpdateClientIdEntry(cpputils::Deserializer *deserializer);
+            static void _serializeLastUpdateClientIdEntry(cpputils::Serializer *serializer, const std::pair<BlockId, uint32_t> &entry);
+
+            DISALLOW_COPY_AND_ASSIGN(KnownBlockVersions);
+        };
+
+    }
+}
+
+
+#endif

src/blockstore/implementations/low2highlevel/LowToHighLevelBlock.cpp

@@ -0,0 +1,83 @@
+#include "LowToHighLevelBlock.h"
+
+using boost::optional;
+using boost::none;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using cpputils::Data;
+namespace DataUtils = cpputils::DataUtils;
+using std::unique_lock;
+using std::mutex;
+
+namespace blockstore {
+namespace lowtohighlevel {
+
+optional<unique_ref<LowToHighLevelBlock>> LowToHighLevelBlock::TryCreateNew(BlockStore2 *baseBlockStore, const BlockId &blockId, Data data) {
+  bool success = baseBlockStore->tryCreate(blockId, data);
+  if (!success) {
+    return none;
+  }
+
+  return make_unique_ref<LowToHighLevelBlock>(blockId, std::move(data), baseBlockStore);
+}
+
+unique_ref<LowToHighLevelBlock> LowToHighLevelBlock::Overwrite(BlockStore2 *baseBlockStore, const BlockId &blockId, Data data) {
+  baseBlockStore->store(blockId, data); // TODO Does it make sense to not store here, but only write back in the destructor of LowToHighLevelBlock? Also: What about tryCreate?
+  return make_unique_ref<LowToHighLevelBlock>(blockId, std::move(data), baseBlockStore);
+}
+
+optional<unique_ref<LowToHighLevelBlock>> LowToHighLevelBlock::Load(BlockStore2 *baseBlockStore, const BlockId &blockId) {
+  optional<Data> loadedData = baseBlockStore->load(blockId);
+  if (loadedData == none) {
+    return none;
+  }
+  return make_unique_ref<LowToHighLevelBlock>(blockId, std::move(*loadedData), baseBlockStore);
+}
+
+LowToHighLevelBlock::LowToHighLevelBlock(const BlockId &blockId, Data data, BlockStore2 *baseBlockStore)
+    :Block(blockId),
+     _baseBlockStore(baseBlockStore),
+     _data(std::move(data)),
+     _dataChanged(false),
+     _mutex() {
+}
+
+LowToHighLevelBlock::~LowToHighLevelBlock() {
+  unique_lock<mutex> lock(_mutex);
+  _storeToBaseBlock();
+}
+
+const void *LowToHighLevelBlock::data() const {
+  return _data.data();
+}
+
+void LowToHighLevelBlock::write(const void *source, uint64_t offset, uint64_t count) {
+  ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition
+  std::memcpy(_data.dataOffset(offset), source, count);
+  _dataChanged = true;
+}
+
+void LowToHighLevelBlock::flush() {
+  unique_lock<mutex> lock(_mutex);
+  _storeToBaseBlock();
+}
+
+size_t LowToHighLevelBlock::size() const {
+  return _data.size();
+}
+
+void LowToHighLevelBlock::resize(size_t newSize) {
+  _data = DataUtils::resize(_data, newSize);
+  _dataChanged = true;
+}
+
+void LowToHighLevelBlock::_storeToBaseBlock() {
+  if (_dataChanged) {
+    _baseBlockStore->store(blockId(), _data);
+    _dataChanged = false;
+  }
+}
+
+
+}
+}

src/blockstore/implementations/low2highlevel/LowToHighLevelBlock.h

@@ -0,0 +1,54 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_LOWTOHIGHLEVEL_LOWTOHIGHLEVELBLOCK_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_LOWTOHIGHLEVEL_LOWTOHIGHLEVELBLOCK_H_
+
+#include "../../interface/Block.h"
+#include <cpp-utils/data/Data.h>
+#include "../../interface/BlockStore.h"
+#include "../../interface/BlockStore2.h"
+
+#include <cpp-utils/macros.h>
+#include <memory>
+#include <iostream>
+#include <boost/optional.hpp>
+#include <cpp-utils/crypto/symmetric/Cipher.h>
+#include <cpp-utils/assert/assert.h>
+#include <cpp-utils/data/DataUtils.h>
+#include <mutex>
+#include <cpp-utils/logging/logging.h>
+#include "LowToHighLevelBlockStore.h"
+
+namespace blockstore {
+namespace lowtohighlevel {
+
+class LowToHighLevelBlock final: public Block {
+public:
+  static boost::optional<cpputils::unique_ref<LowToHighLevelBlock>> TryCreateNew(BlockStore2 *baseBlockStore, const BlockId &blockId, cpputils::Data data);
+  static cpputils::unique_ref<LowToHighLevelBlock> Overwrite(BlockStore2 *baseBlockStore, const BlockId &blockId, cpputils::Data data);
+  static boost::optional<cpputils::unique_ref<LowToHighLevelBlock>> Load(BlockStore2 *baseBlockStore, const BlockId &blockId);
+
+  LowToHighLevelBlock(const BlockId &blockId, cpputils::Data data, BlockStore2 *baseBlockStore);
+  ~LowToHighLevelBlock();
+
+  const void *data() const override;
+  void write(const void *source, uint64_t offset, uint64_t count) override;
+  void flush() override;
+
+  size_t size() const override;
+  void resize(size_t newSize) override;
+
+private:
+  BlockStore2 *_baseBlockStore;
+  cpputils::Data _data;
+  bool _dataChanged;
+  std::mutex _mutex;
+
+  void _storeToBaseBlock();
+
+  DISALLOW_COPY_AND_ASSIGN(LowToHighLevelBlock);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.cpp

@@ -0,0 +1,70 @@
+#include <unordered_set>
+#include "LowToHighLevelBlockStore.h"
+#include "LowToHighLevelBlock.h"
+
+using cpputils::unique_ref;
+using cpputils::Data;
+using boost::none;
+using boost::optional;
+using std::string;
+
+namespace blockstore {
+namespace lowtohighlevel {
+
+LowToHighLevelBlockStore::LowToHighLevelBlockStore(unique_ref<BlockStore2> baseBlockStore)
+    : _baseBlockStore(std::move(baseBlockStore)) {
+}
+
+BlockId LowToHighLevelBlockStore::createBlockId() {
+    // TODO Is this the right way?
+    return BlockId::Random();
+}
+
+optional<unique_ref<Block>> LowToHighLevelBlockStore::tryCreate(const BlockId &blockId, Data data) {
+    //TODO Easier implementation? This is only so complicated because of the cast LowToHighLevelBlock -> Block
+    auto result = LowToHighLevelBlock::TryCreateNew(_baseBlockStore.get(), blockId, std::move(data));
+    if (result == none) {
+        return none;
+    }
+    return unique_ref<Block>(std::move(*result));
+}
+
+unique_ref<Block> LowToHighLevelBlockStore::overwrite(const BlockId &blockId, Data data) {
+    return unique_ref<Block>(
+        LowToHighLevelBlock::Overwrite(_baseBlockStore.get(), blockId, std::move(data))
+    );
+}
+
+optional<unique_ref<Block>> LowToHighLevelBlockStore::load(const BlockId &blockId) {
+    auto result = optional<unique_ref<Block>>(LowToHighLevelBlock::Load(_baseBlockStore.get(), blockId));
+    if (result == none) {
+      return none;
+    }
+    return unique_ref<Block>(std::move(*result));
+}
+
+void LowToHighLevelBlockStore::remove(const BlockId &blockId) {
+    bool success = _baseBlockStore->remove(blockId);
+    if (!success) {
+        throw std::runtime_error("Couldn't delete block with id " + blockId.ToString());
+    }
+}
+
+uint64_t LowToHighLevelBlockStore::numBlocks() const {
+    return _baseBlockStore->numBlocks();
+}
+
+uint64_t LowToHighLevelBlockStore::estimateNumFreeBytes() const {
+    return _baseBlockStore->estimateNumFreeBytes();
+}
+
+uint64_t LowToHighLevelBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+    return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
+}
+
+void LowToHighLevelBlockStore::forEachBlock(std::function<void (const BlockId &)> callback) const {
+    _baseBlockStore->forEachBlock(std::move(callback));
+}
+
+}
+}

src/blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h

@@ -0,0 +1,42 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_LOWTOHIGHLEVEL_LOWTOHIGHLEVELBLOCKSTORE_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_LOWTOHIGHLEVEL_LOWTOHIGHLEVELBLOCKSTORE_H_
+
+#include "../../interface/BlockStore.h"
+#include "../../interface/BlockStore2.h"
+#include <cpp-utils/macros.h>
+#include <cpp-utils/pointer/cast.h>
+#include <iostream>
+
+// TODO Think each function through and make sure it's as performant
+//      to use LowToHighLevelBlockStore<OnDiskBlockStore2> as to use
+//      OnDiskBlockStore directly (i.e. no additional stores/loads from the disk)
+//      (same for other base block stores)
+
+namespace blockstore {
+namespace lowtohighlevel {
+
+class LowToHighLevelBlockStore final: public BlockStore {
+public:
+  LowToHighLevelBlockStore(cpputils::unique_ref<BlockStore2> baseBlockStore);
+
+  BlockId createBlockId() override;
+  boost::optional<cpputils::unique_ref<Block>> tryCreate(const BlockId &blockId, cpputils::Data data) override;
+  cpputils::unique_ref<Block> overwrite(const blockstore::BlockId &blockId, cpputils::Data data) override;
+  boost::optional<cpputils::unique_ref<Block>> load(const BlockId &blockId) override;
+  void remove(const BlockId &blockId) override;
+  uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
+  void forEachBlock(std::function<void (const BlockId &)> callback) const override;
+
+private:
+  cpputils::unique_ref<BlockStore2> _baseBlockStore;
+
+  DISALLOW_COPY_AND_ASSIGN(LowToHighLevelBlockStore);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/mock/MockBlock.cpp

@@ -0,0 +1,18 @@
+#include "MockBlock.h"
+#include "MockBlockStore.h"
+
+namespace blockstore {
+    namespace mock {
+
+        void MockBlock::write(const void *source, uint64_t offset, uint64_t size) {
+            _blockStore->_increaseNumWrittenBlocks(blockId());
+            return _baseBlock->write(source, offset, size);
+        }
+
+        void MockBlock::resize(size_t newSize) {
+            _blockStore->_increaseNumResizedBlocks(blockId());
+            return _baseBlock->resize(newSize);
+        }
+
+    }
+}

src/blockstore/implementations/mock/MockBlock.h

@@ -0,0 +1,50 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_MOCK_MOCKBLOCK_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_MOCK_MOCKBLOCK_H_
+
+#include <blockstore/interface/Block.h>
+#include <cpp-utils/pointer/unique_ref.h>
+
+namespace blockstore {
+    namespace mock {
+
+        class MockBlockStore;
+
+        class MockBlock final : public blockstore::Block {
+        public:
+            MockBlock(cpputils::unique_ref<blockstore::Block> baseBlock, MockBlockStore *blockStore)
+                    :Block(baseBlock->blockId()), _baseBlock(std::move(baseBlock)), _blockStore(blockStore) {
+            }
+
+            const void *data() const override {
+              return _baseBlock->data();
+            }
+
+            void write(const void *source, uint64_t offset, uint64_t size) override;
+
+            void flush() override {
+              return _baseBlock->flush();
+            }
+
+            size_t size() const override {
+              return _baseBlock->size();
+            }
+
+            void resize(size_t newSize) override;
+
+            cpputils::unique_ref<blockstore::Block> releaseBaseBlock() {
+              return std::move(_baseBlock);
+            }
+
+        private:
+            cpputils::unique_ref<blockstore::Block> _baseBlock;
+            MockBlockStore *_blockStore;
+
+            DISALLOW_COPY_AND_ASSIGN(MockBlock);
+        };
+
+
+    }
+}
+
+#endif

src/blockstore/implementations/mock/MockBlockStore.cpp

@@ -0,0 +1 @@
+#include "MockBlockStore.h"

src/blockstore/implementations/mock/MockBlockStore.h

@@ -0,0 +1,157 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_MOCK_MOCKBLOCKSTORE_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_MOCK_MOCKBLOCKSTORE_H_
+
+#include <blockstore/implementations/testfake/FakeBlockStore.h>
+#include <mutex>
+#include "MockBlock.h"
+
+namespace blockstore {
+    namespace mock {
+
+        /**
+         * This is a blockstore that counts the number of loaded, resized, written, ... blocks.
+         * It is used for testing that operations only access few blocks (performance tests).
+         */
+        class MockBlockStore final : public BlockStore {
+        public:
+            MockBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore = cpputils::make_unique_ref<testfake::FakeBlockStore>())
+                    : _mutex(), _baseBlockStore(std::move(baseBlockStore)), _loadedBlocks(), _createdBlocks(0), _writtenBlocks(), _resizedBlocks(), _removedBlocks() {
+            }
+
+            BlockId createBlockId() override {
+                return _baseBlockStore->createBlockId();
+            }
+
+            boost::optional<cpputils::unique_ref<Block>> tryCreate(const BlockId &blockId, cpputils::Data data) override {
+                _increaseNumCreatedBlocks();
+                auto base = _baseBlockStore->tryCreate(blockId, std::move(data));
+                if (base == boost::none) {
+                    return boost::none;
+                }
+                return boost::optional<cpputils::unique_ref<Block>>(cpputils::make_unique_ref<MockBlock>(std::move(*base), this));
+            }
+
+            boost::optional<cpputils::unique_ref<Block>> load(const BlockId &blockId) override {
+                _increaseNumLoadedBlocks(blockId);
+                auto base = _baseBlockStore->load(blockId);
+                if (base == boost::none) {
+                    return boost::none;
+                }
+                return boost::optional<cpputils::unique_ref<Block>>(cpputils::make_unique_ref<MockBlock>(std::move(*base), this));
+            }
+
+            cpputils::unique_ref<Block> overwrite(const BlockId &blockId, cpputils::Data data) override {
+                _increaseNumWrittenBlocks(blockId);
+                return _baseBlockStore->overwrite(blockId, std::move(data));
+            }
+
+            void remove(const BlockId &blockId) override {
+                _increaseNumRemovedBlocks(blockId);
+                return _baseBlockStore->remove(blockId);
+            }
+
+            uint64_t numBlocks() const override {
+                return _baseBlockStore->numBlocks();
+            }
+
+            uint64_t estimateNumFreeBytes() const override {
+                return _baseBlockStore->estimateNumFreeBytes();
+            }
+
+            uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override {
+                return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
+            }
+
+            void forEachBlock(std::function<void(const BlockId &)> callback) const override {
+                return _baseBlockStore->forEachBlock(callback);
+            }
+
+            void remove(cpputils::unique_ref<Block> block) override {
+                _increaseNumRemovedBlocks(block->blockId());
+                auto mockBlock = cpputils::dynamic_pointer_move<MockBlock>(block);
+                ASSERT(mockBlock != boost::none, "Wrong block type");
+                return _baseBlockStore->remove((*mockBlock)->releaseBaseBlock());
+            }
+
+            void resetCounters() {
+                _loadedBlocks = {};
+                _createdBlocks = 0;
+                _removedBlocks = {};
+                _resizedBlocks = {};
+                _writtenBlocks = {};
+            }
+
+            uint64_t createdBlocks() const {
+                return _createdBlocks;
+            }
+
+            const std::vector<BlockId> &loadedBlocks() const {
+                return _loadedBlocks;
+            }
+
+            const std::vector<BlockId> &removedBlocks() const {
+                return _removedBlocks;
+            }
+
+            const std::vector<BlockId> &resizedBlocks() const {
+                return _resizedBlocks;
+            }
+
+            const std::vector<BlockId> &writtenBlocks() const {
+                return _writtenBlocks;
+            }
+
+            std::vector<BlockId> distinctWrittenBlocks() const {
+                std::vector<BlockId> result(_writtenBlocks);
+                std::sort(result.begin(), result.end(), [](const BlockId &lhs, const BlockId &rhs) {
+                    return std::memcmp(lhs.data().data(), rhs.data().data(), lhs.BINARY_LENGTH) < 0;
+                });
+                result.erase(std::unique(result.begin(), result.end() ), result.end());
+                return result;
+            }
+
+        private:
+            void _increaseNumCreatedBlocks() {
+                std::unique_lock<std::mutex> lock(_mutex);
+                _createdBlocks += 1;
+            }
+
+            void _increaseNumLoadedBlocks(const BlockId &blockId) {
+                std::unique_lock<std::mutex> lock(_mutex);
+                _loadedBlocks.push_back(blockId);
+            }
+
+            void _increaseNumRemovedBlocks(const BlockId &blockId) {
+                std::unique_lock<std::mutex> lock(_mutex);
+                _removedBlocks.push_back(blockId);
+            }
+
+            void _increaseNumResizedBlocks(const BlockId &blockId) {
+                std::unique_lock<std::mutex> lock(_mutex);
+                _resizedBlocks.push_back(blockId);
+            }
+
+            void _increaseNumWrittenBlocks(const BlockId &blockId) {
+                std::unique_lock<std::mutex> lock(_mutex);
+                _writtenBlocks.push_back(blockId);
+            }
+
+            friend class MockBlock;
+
+            std::mutex _mutex;
+            cpputils::unique_ref<BlockStore> _baseBlockStore;
+
+            std::vector<BlockId> _loadedBlocks;
+            uint64_t _createdBlocks;
+            std::vector<BlockId> _writtenBlocks;
+            std::vector<BlockId> _resizedBlocks;
+            std::vector<BlockId> _removedBlocks;
+
+            DISALLOW_COPY_AND_ASSIGN(MockBlockStore);
+        };
+
+    }
+}
+
+#endif

src/blockstore/implementations/ondisk/OnDiskBlock.cpp

@@ -1,175 +0,0 @@
-#include <cstring>
-#include <fstream>
-#include <boost/filesystem.hpp>
-#include "OnDiskBlock.h"
-#include "OnDiskBlockStore.h"
-#include "../../utils/FileDoesntExistException.h"
-#include <cpp-utils/data/DataUtils.h>
-#include <cpp-utils/assert/assert.h>
-#include <cpp-utils/logging/logging.h>
-
-using std::istream;
-using std::ostream;
-using std::ifstream;
-using std::ofstream;
-using std::ios;
-using std::string;
-using cpputils::Data;
-using cpputils::make_unique_ref;
-using cpputils::unique_ref;
-using boost::optional;
-using boost::none;
-
-namespace bf = boost::filesystem;
-using namespace cpputils::logging;
-
-namespace blockstore {
-namespace ondisk {
-
-const string OnDiskBlock::FORMAT_VERSION_HEADER_PREFIX = "cryfs;block;";
-const string OnDiskBlock::FORMAT_VERSION_HEADER = OnDiskBlock::FORMAT_VERSION_HEADER_PREFIX + "0";
-
-OnDiskBlock::OnDiskBlock(const Key &key, const bf::path &filepath, Data data)
- : Block(key), _filepath(filepath), _data(std::move(data)), _dataChanged(false), _mutex() {
-}
-
-OnDiskBlock::~OnDiskBlock() {
-  flush();
-}
-
-const void *OnDiskBlock::data() const {
-  return _data.data();
-}
-
-void OnDiskBlock::write(const void *source, uint64_t offset, uint64_t size) {
-  ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
-  std::memcpy(_data.dataOffset(offset), source, size);
-  _dataChanged = true;
-}
-
-size_t OnDiskBlock::size() const {
-  return _data.size();
-}
-
-void OnDiskBlock::resize(size_t newSize) {
-  _data = cpputils::DataUtils::resize(std::move(_data), newSize);
-  _dataChanged = true;
-}
-
-bf::path OnDiskBlock::_getFilepath(const bf::path &rootdir, const Key &key) {
-  string keyStr = key.ToString();
-  return rootdir / keyStr.substr(0,3) / keyStr.substr(3);
-}
-
-optional<unique_ref<OnDiskBlock>> OnDiskBlock::LoadFromDisk(const bf::path &rootdir, const Key &key) {
-  auto filepath = _getFilepath(rootdir, key);
-  try {
-    boost::optional<Data> data = _loadFromDisk(filepath);
-    if (data == none) {
-      return none;
-    }
-    return make_unique_ref<OnDiskBlock>(key, filepath, std::move(*data));
-  } catch (const FileDoesntExistException &e) {
-    return none;
-  }
-}
-
-optional<unique_ref<OnDiskBlock>> OnDiskBlock::CreateOnDisk(const bf::path &rootdir, const Key &key, Data data) {
-  auto filepath = _getFilepath(rootdir, key);
-  bf::create_directory(filepath.parent_path());
-  if (bf::exists(filepath)) {
-    return none;
-  }
-
-  auto block = make_unique_ref<OnDiskBlock>(key, filepath, std::move(data));
-  block->_storeToDisk();
-  return std::move(block);
-}
-
-void OnDiskBlock::RemoveFromDisk(const bf::path &rootdir, const Key &key) {
-  auto filepath = _getFilepath(rootdir, key);
-  ASSERT(bf::is_regular_file(filepath), "Block not found on disk");
-  bool retval = bf::remove(filepath);
-  if (!retval) {
-    LOG(ERROR, "Couldn't find block {} to remove", key.ToString());
-  }
-  if (bf::is_empty(filepath.parent_path())) {
-    bf::remove(filepath.parent_path());
-  }
-}
-
-void OnDiskBlock::_storeToDisk() const {
-  std::ofstream file(_filepath.c_str(), std::ios::binary | std::ios::trunc);
-  if (!file.good()) {
-    throw std::runtime_error("Could not open file for writing");
-  }
-  file.write(FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
-  if (!file.good()) {
-    throw std::runtime_error("Error writing block header");
-  }
-  _data.StoreToStream(file);
-  if (!file.good()) {
-    throw std::runtime_error("Error writing block data");
-  }
-}
-
-optional<Data> OnDiskBlock::_loadFromDisk(const bf::path &filepath) {
-  //If it isn't a file, ifstream::good() would return false. We still need this extra check
-  //upfront, because ifstream::good() doesn't crash if we give it the path of a directory
-  //instead the path of a file.
-  if(!bf::is_regular_file(filepath)) {
-    return none;
-  }
-  ifstream file(filepath.c_str(), ios::binary);
-  if (!file.good()) {
-    return none;
-  }
-  _checkHeader(&file);
-  Data result = Data::LoadFromStream(file);
-  //TODO With newer compilers, "return result;" would be enough
-  return boost::optional<Data>(std::move(result));
-}
-
-void OnDiskBlock::_checkHeader(istream *str) {
-  Data header(formatVersionHeaderSize());
-  str->read(reinterpret_cast<char*>(header.data()), formatVersionHeaderSize());
-  if (!_isAcceptedCryfsHeader(header)) {
-    if (_isOtherCryfsHeader(header)) {
-      throw std::runtime_error("This block is not supported yet. Maybe it was created with a newer version of CryFS?");
-    } else {
-      throw std::runtime_error("This is not a valid block.");
-    }
-  }
-}
-
-bool OnDiskBlock::_isAcceptedCryfsHeader(const Data &data) {
-  ASSERT(data.size() == formatVersionHeaderSize(), "We extracted the wrong header size from the block.");
-  return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
-}
-
-bool OnDiskBlock::_isOtherCryfsHeader(const Data &data) {
-  ASSERT(data.size() >= FORMAT_VERSION_HEADER_PREFIX.size(), "We extracted the wrong header size from the block.");
-  return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER_PREFIX.c_str(), FORMAT_VERSION_HEADER_PREFIX.size());
-}
-
-unsigned int OnDiskBlock::formatVersionHeaderSize() {
-  return FORMAT_VERSION_HEADER.size() + 1; // +1 because of the null byte
-}
-
-void OnDiskBlock::flush() {
-  std::unique_lock<std::mutex> lock(_mutex);
-  if (_dataChanged) {
-    _storeToDisk();
-    _dataChanged = false;
-  }
-}
-
-uint64_t OnDiskBlock::blockSizeFromPhysicalBlockSize(uint64_t blockSize) {
-  if(blockSize <= formatVersionHeaderSize()) {
-    return 0;
-  }
-  return blockSize - formatVersionHeaderSize();
-}
-
-}
-}

src/blockstore/implementations/ondisk/OnDiskBlock.h

@@ -1,61 +0,0 @@
-#pragma once
-#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
-#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
-
-#include <boost/filesystem/path.hpp>
-#include "../../interface/Block.h"
-#include <cpp-utils/data/Data.h>
-#include <iostream>
-
-#include <cpp-utils/pointer/unique_ref.h>
-#include <mutex>
-
-namespace blockstore {
-namespace ondisk {
-class OnDiskBlockStore;
-
-class OnDiskBlock final: public Block {
-public:
-  OnDiskBlock(const Key &key, const boost::filesystem::path &filepath, cpputils::Data data);
-  ~OnDiskBlock();
-
-  static const std::string FORMAT_VERSION_HEADER_PREFIX;
-  static const std::string FORMAT_VERSION_HEADER;
-  static unsigned int formatVersionHeaderSize();
-  static uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize);
-
-  static boost::optional<cpputils::unique_ref<OnDiskBlock>> LoadFromDisk(const boost::filesystem::path &rootdir, const Key &key);
-  static boost::optional<cpputils::unique_ref<OnDiskBlock>> CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data);
-  static void RemoveFromDisk(const boost::filesystem::path &rootdir, const Key &key);
-
-  const void *data() const override;
-  void write(const void *source, uint64_t offset, uint64_t size) override;
-
-  void flush() override;
-
-  size_t size() const override;
-  void resize(size_t newSize) override;
-
-private:
-
-  static bool _isAcceptedCryfsHeader(const cpputils::Data &data);
-  static bool _isOtherCryfsHeader(const cpputils::Data &data);
-  static void _checkHeader(std::istream *str);
-  static boost::filesystem::path _getFilepath(const boost::filesystem::path &rootdir, const Key &key);
-
-  const boost::filesystem::path _filepath;
-  cpputils::Data _data;
-  bool _dataChanged;
-
-  static boost::optional<cpputils::Data> _loadFromDisk(const boost::filesystem::path &filepath);
-  void _storeToDisk() const;
-
-  std::mutex _mutex;
-
-  DISALLOW_COPY_AND_ASSIGN(OnDiskBlock);
-};
-
-}
-}
-
-#endif

src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp

@@ -1,99 +0,0 @@
-#include "OnDiskBlock.h"
-#include "OnDiskBlockStore.h"
-#include <sys/statvfs.h>
-
-using std::string;
-using cpputils::Data;
-using cpputils::unique_ref;
-using boost::optional;
-using boost::none;
-using std::vector;
-
-namespace bf = boost::filesystem;
-
-namespace blockstore {
-namespace ondisk {
-
-OnDiskBlockStore::OnDiskBlockStore(const boost::filesystem::path &rootdir)
- : _rootdir(rootdir) {
-  if (!bf::exists(rootdir)) {
-    throw std::runtime_error("Base directory not found");
-  }
-  if (!bf::is_directory(rootdir)) {
-      throw std::runtime_error("Base directory is not a directory");
-  }
-  //TODO Test for read access, write access, enter (x) access, and throw runtime_error in case
-#ifndef CRYFS_NO_COMPATIBILITY
-  _migrateBlockStore();
-#endif
-}
-
-#ifndef CRYFS_NO_COMPATIBILITY
-void OnDiskBlockStore::_migrateBlockStore() {
-  vector<string> blocksToMigrate;
-  for (auto entry = bf::directory_iterator(_rootdir); entry != bf::directory_iterator(); ++entry) {
-    if (bf::is_regular_file(entry->path()) && _isValidBlockKey(entry->path().filename().native())) {
-      blocksToMigrate.push_back(entry->path().filename().native());
-    }
-  }
-  if (blocksToMigrate.size() != 0) {
-    std::cout << "Migrating CryFS filesystem..." << std::flush;
-    for (auto key : blocksToMigrate) {
-      Key::FromString(key); // Assert that it can be parsed as a key
-      string dir = key.substr(0, 3);
-      string file = key.substr(3);
-      bf::create_directory(_rootdir / dir);
-      bf::rename(_rootdir / key, _rootdir / dir / file);
-    }
-    std::cout << "done" << std::endl;
-  }
-}
-
-bool OnDiskBlockStore::_isValidBlockKey(const string &key) {
-  return key.size() == 32 && key.find_first_not_of("0123456789ABCDEF") == string::npos;
-}
-#endif
-
-//TODO Do I have to lock tryCreate/remove and/or load? Or does ParallelAccessBlockStore take care of that?
-
-optional<unique_ref<Block>> OnDiskBlockStore::tryCreate(const Key &key, Data data) {
-  //TODO Easier implementation? This is only so complicated because of the cast OnDiskBlock -> Block
-  auto result = OnDiskBlock::CreateOnDisk(_rootdir, key, std::move(data));
-  if (result == boost::none) {
-    return boost::none;
-  }
-  return unique_ref<Block>(std::move(*result));
-}
-
-optional<unique_ref<Block>> OnDiskBlockStore::load(const Key &key) {
-  return optional<unique_ref<Block>>(OnDiskBlock::LoadFromDisk(_rootdir, key));
-}
-
-void OnDiskBlockStore::remove(unique_ref<Block> block) {
-  Key key = block->key();
-  cpputils::destruct(std::move(block));
-  OnDiskBlock::RemoveFromDisk(_rootdir, key);
-}
-
-uint64_t OnDiskBlockStore::numBlocks() const {
-  uint64_t count = 0;
-  for (auto entry = bf::directory_iterator(_rootdir); entry != bf::directory_iterator(); ++entry) {
-    if (bf::is_directory(entry->path())) {
-      count += std::distance(bf::directory_iterator(entry->path()), bf::directory_iterator());
-    }
-  }
-  return count;
-}
-
-uint64_t OnDiskBlockStore::estimateNumFreeBytes() const {
-  struct statvfs stat;
-  ::statvfs(_rootdir.c_str(), &stat);
-  return stat.f_bsize*stat.f_bavail;
-}
-
-uint64_t OnDiskBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
-  return OnDiskBlock::blockSizeFromPhysicalBlockSize(blockSize);
-}
-
-}
-}

src/blockstore/implementations/ondisk/OnDiskBlockStore.h

@@ -1,38 +0,0 @@
-#pragma once
-#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
-#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
-
-#include <boost/filesystem.hpp>
-#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
-
-#include <cpp-utils/macros.h>
-
-namespace blockstore {
-namespace ondisk {
-
-class OnDiskBlockStore final: public BlockStoreWithRandomKeys {
-public:
-  explicit OnDiskBlockStore(const boost::filesystem::path &rootdir);
-
-  boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
-  boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
-  //TODO Can we make this faster by allowing to delete blocks by only having theiy Key? So we wouldn't have to load it first?
-  void remove(cpputils::unique_ref<Block> block) override;
-  uint64_t numBlocks() const override;
-  uint64_t estimateNumFreeBytes() const override;
-  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
-
-private:
-  const boost::filesystem::path _rootdir;
-#ifndef CRYFS_NO_COMPATIBILITY
-  void _migrateBlockStore();
-  bool _isValidBlockKey(const std::string &key);
-#endif
-
-  DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore);
-};
-
-}
-}
-
-#endif

src/blockstore/implementations/ondisk/OnDiskBlockStore2.cpp

@@ -0,0 +1,126 @@
+#include "OnDiskBlockStore2.h"
+#include <boost/filesystem.hpp>
+#include <cpp-utils/system/diskspace.h>
+
+using std::string;
+using boost::optional;
+using boost::none;
+using cpputils::Data;
+
+namespace blockstore {
+namespace ondisk {
+
+const string OnDiskBlockStore2::FORMAT_VERSION_HEADER_PREFIX = "cryfs;block;";
+const string OnDiskBlockStore2::FORMAT_VERSION_HEADER = OnDiskBlockStore2::FORMAT_VERSION_HEADER_PREFIX + "0";
+
+boost::filesystem::path OnDiskBlockStore2::_getFilepath(const BlockId &blockId) const {
+  std::string blockIdStr = blockId.ToString();
+  return _rootDir / blockIdStr.substr(0,3) / blockIdStr.substr(3);
+}
+
+Data OnDiskBlockStore2::_checkAndRemoveHeader(const Data &data) {
+  if (!_isAcceptedCryfsHeader(data)) {
+    if (_isOtherCryfsHeader(data)) {
+      throw std::runtime_error("This block is not supported yet. Maybe it was created with a newer version of CryFS?");
+    } else {
+      throw std::runtime_error("This is not a valid block.");
+    }
+  }
+  Data result(data.size() - formatVersionHeaderSize());
+  std::memcpy(result.data(), data.dataOffset(formatVersionHeaderSize()), result.size());
+  return result;
+}
+
+bool OnDiskBlockStore2::_isAcceptedCryfsHeader(const Data &data) {
+  return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
+}
+
+bool OnDiskBlockStore2::_isOtherCryfsHeader(const Data &data) {
+  return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER_PREFIX.c_str(), FORMAT_VERSION_HEADER_PREFIX.size());
+}
+
+unsigned int OnDiskBlockStore2::formatVersionHeaderSize() {
+  return FORMAT_VERSION_HEADER.size() + 1; // +1 because of the null byte
+}
+
+OnDiskBlockStore2::OnDiskBlockStore2(const boost::filesystem::path& path)
+    : _rootDir(path) {}
+
+bool OnDiskBlockStore2::tryCreate(const BlockId &blockId, const Data &data) {
+  auto filepath = _getFilepath(blockId);
+  if (boost::filesystem::exists(filepath)) {
+    return false;
+  }
+
+  store(blockId, data);
+  return true;
+}
+
+bool OnDiskBlockStore2::remove(const BlockId &blockId) {
+  auto filepath = _getFilepath(blockId);
+  if (!boost::filesystem::is_regular_file(filepath)) { // TODO Is this branch necessary?
+    return false;
+  }
+  bool retval = boost::filesystem::remove(filepath);
+  if (!retval) {
+    cpputils::logging::LOG(cpputils::logging::ERR, "Couldn't find block {} to remove", blockId.ToString());
+    return false;
+  }
+  if (boost::filesystem::is_empty(filepath.parent_path())) {
+    boost::filesystem::remove(filepath.parent_path());
+  }
+  return true;
+}
+
+optional<Data> OnDiskBlockStore2::load(const BlockId &blockId) const {
+  auto fileContent = Data::LoadFromFile(_getFilepath(blockId));
+  if (fileContent == none) {
+    return boost::none;
+  }
+  return _checkAndRemoveHeader(*fileContent);
+}
+
+void OnDiskBlockStore2::store(const BlockId &blockId, const Data &data) {
+  Data fileContent(formatVersionHeaderSize() + data.size());
+  std::memcpy(fileContent.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
+  std::memcpy(fileContent.dataOffset(formatVersionHeaderSize()), data.data(), data.size());
+  auto filepath = _getFilepath(blockId);
+  boost::filesystem::create_directory(filepath.parent_path()); // TODO Instead create all of them once at fs creation time?
+  fileContent.StoreToFile(filepath);
+}
+
+uint64_t OnDiskBlockStore2::numBlocks() const {
+  uint64_t count = 0;
+  for (auto prefixDir = boost::filesystem::directory_iterator(_rootDir); prefixDir != boost::filesystem::directory_iterator(); ++prefixDir) {
+    if (boost::filesystem::is_directory(prefixDir->path())) {
+      count += std::distance(boost::filesystem::directory_iterator(prefixDir->path()), boost::filesystem::directory_iterator());
+    }
+  }
+  return count;
+}
+
+uint64_t OnDiskBlockStore2::estimateNumFreeBytes() const {
+	return cpputils::free_disk_space_in_bytes(_rootDir);
+}
+
+uint64_t OnDiskBlockStore2::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  if(blockSize <= formatVersionHeaderSize()) {
+    return 0;
+  }
+  return blockSize - formatVersionHeaderSize();
+}
+
+void OnDiskBlockStore2::forEachBlock(std::function<void (const BlockId &)> callback) const {
+  for (auto prefixDir = boost::filesystem::directory_iterator(_rootDir); prefixDir != boost::filesystem::directory_iterator(); ++prefixDir) {
+    if (boost::filesystem::is_directory(prefixDir->path())) {
+      std::string blockIdPrefix = prefixDir->path().filename().string();
+      for (auto block = boost::filesystem::directory_iterator(prefixDir->path()); block != boost::filesystem::directory_iterator(); ++block) {
+        std::string blockIdPostfix = block->path().filename().string();
+        callback(BlockId::FromString(blockIdPrefix + blockIdPostfix));
+      }
+    }
+  }
+}
+
+}
+}

src/blockstore/implementations/ondisk/OnDiskBlockStore2.h

@@ -0,0 +1,45 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE2_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE2_H_
+
+#include "../../interface/BlockStore2.h"
+#include <boost/filesystem/path.hpp>
+#include <cpp-utils/macros.h>
+#include <cpp-utils/pointer/unique_ref.h>
+#include <cpp-utils/logging/logging.h>
+
+namespace blockstore {
+namespace ondisk {
+
+class OnDiskBlockStore2 final: public BlockStore2 {
+public:
+  explicit OnDiskBlockStore2(const boost::filesystem::path& path);
+
+  bool tryCreate(const BlockId &blockId, const cpputils::Data &data) override;
+  bool remove(const BlockId &blockId) override;
+  boost::optional<cpputils::Data> load(const BlockId &blockId) const override;
+  void store(const BlockId &blockId, const cpputils::Data &data) override;
+  uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
+  void forEachBlock(std::function<void (const BlockId &)> callback) const override;
+
+private:
+  boost::filesystem::path _rootDir;
+
+  static const std::string FORMAT_VERSION_HEADER_PREFIX;
+  static const std::string FORMAT_VERSION_HEADER;
+
+  boost::filesystem::path _getFilepath(const BlockId &blockId) const;
+  static cpputils::Data _checkAndRemoveHeader(const cpputils::Data &data);
+  static bool _isAcceptedCryfsHeader(const cpputils::Data &data);
+  static bool _isOtherCryfsHeader(const cpputils::Data &data);
+  static unsigned int formatVersionHeaderSize();
+
+  DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore2);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/parallelaccess/BlockRef.h

@@ -11,10 +11,10 @@ namespace blockstore {
 namespace parallelaccess {
 class ParallelAccessBlockStore;
 
-class BlockRef final: public Block, public parallelaccessstore::ParallelAccessStore<Block, BlockRef, Key>::ResourceRefBase {
+class BlockRef final: public Block, public parallelaccessstore::ParallelAccessStore<Block, BlockRef, BlockId>::ResourceRefBase {
 public:
-  //TODO Unneccessarily storing Key twice here (in parent class and in _baseBlock).
+  //TODO Unneccessarily storing BlockId twice here (in parent class and in _baseBlock).
-  explicit BlockRef(Block *baseBlock): Block(baseBlock->key()), _baseBlock(baseBlock) {}
+  explicit BlockRef(Block *baseBlock): Block(baseBlock->blockId()), _baseBlock(baseBlock) {}
 
   const void *data() const override {
 	return _baseBlock->data();

src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.cpp

@@ -6,12 +6,12 @@
 #include <cpp-utils/assert/assert.h>
 
 using std::string;
-using std::promise;
 using cpputils::dynamic_pointer_move;
 using cpputils::make_unique_ref;
 using boost::none;
 using cpputils::unique_ref;
 using cpputils::make_unique_ref;
+using cpputils::Data;
 using boost::optional;
 using boost::none;
 
@@ -22,34 +22,52 @@ ParallelAccessBlockStore::ParallelAccessBlockStore(unique_ref<BlockStore> baseBl
  : _baseBlockStore(std::move(baseBlockStore)), _parallelAccessStore(make_unique_ref<ParallelAccessBlockStoreAdapter>(_baseBlockStore.get())) {
 }
 
-Key ParallelAccessBlockStore::createKey() {
+BlockId ParallelAccessBlockStore::createBlockId() {
-  return _baseBlockStore->createKey();
+  return _baseBlockStore->createBlockId();
 }
 
-optional<unique_ref<Block>> ParallelAccessBlockStore::tryCreate(const Key &key, cpputils::Data data) {
+optional<unique_ref<Block>> ParallelAccessBlockStore::tryCreate(const BlockId &blockId, Data data) {
-  ASSERT(!_parallelAccessStore.isOpened(key), ("Key "+key.ToString()+"already exists").c_str());
+  if (_parallelAccessStore.isOpened(blockId)) {
-  auto block = _baseBlockStore->tryCreate(key, std::move(data));
+    return none; // block already exists
+  }
+  auto block = _baseBlockStore->tryCreate(blockId, std::move(data));
   if (block == none) {
 	//TODO Test this code branch
 	return none;
   }
-  return unique_ref<Block>(_parallelAccessStore.add(key, std::move(*block)));
+  return unique_ref<Block>(_parallelAccessStore.add(blockId, std::move(*block)));
 }
 
-optional<unique_ref<Block>> ParallelAccessBlockStore::load(const Key &key) {
+optional<unique_ref<Block>> ParallelAccessBlockStore::load(const BlockId &blockId) {
-  auto block = _parallelAccessStore.load(key);
+  auto block = _parallelAccessStore.load(blockId);
   if (block == none) {
     return none;
   }
   return unique_ref<Block>(std::move(*block));
 }
 
+unique_ref<Block> ParallelAccessBlockStore::overwrite(const BlockId &blockId, Data data) {
+  auto onExists = [&data] (BlockRef *block) {
+      if (block->size() != data.size()) {
+        block->resize(data.size());
+      }
+      block->write(data.data(), 0, data.size());
+  };
+  auto onAdd = [this, blockId, &data] {
+      return _baseBlockStore->overwrite(blockId, data.copy()); // TODO Without copy?
+  };
+  return _parallelAccessStore.loadOrAdd(blockId, onExists, onAdd); // NOLINT (workaround https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82481 )
+}
 
 void ParallelAccessBlockStore::remove(unique_ref<Block> block) {
-  Key key = block->key();
+  BlockId blockId = block->blockId();
   auto block_ref = dynamic_pointer_move<BlockRef>(block);
   ASSERT(block_ref != none, "Block is not a BlockRef");
-  return _parallelAccessStore.remove(key, std::move(*block_ref));
+  return _parallelAccessStore.remove(blockId, std::move(*block_ref));
+}
+
+void ParallelAccessBlockStore::remove(const BlockId &blockId) {
+  return _parallelAccessStore.remove(blockId);
 }
 
 uint64_t ParallelAccessBlockStore::numBlocks() const {
@@ -64,5 +82,9 @@ uint64_t ParallelAccessBlockStore::blockSizeFromPhysicalBlockSize(uint64_t block
   return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
 }
 
+void ParallelAccessBlockStore::forEachBlock(std::function<void (const BlockId &)> callback) const {
+  return _baseBlockStore->forEachBlock(callback);
+}
+
 }
 }

src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h

@@ -15,17 +15,20 @@ class ParallelAccessBlockStore final: public BlockStore {
 public:
   explicit ParallelAccessBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
 
-  Key createKey() override;
+  BlockId createBlockId() override;
-  boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
+  boost::optional<cpputils::unique_ref<Block>> tryCreate(const BlockId &blockId, cpputils::Data data) override;
-  boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
+  boost::optional<cpputils::unique_ref<Block>> load(const BlockId &blockId) override;
-  void remove(cpputils::unique_ref<Block> block) override;
+  cpputils::unique_ref<Block> overwrite(const BlockId &blockId, cpputils::Data data) override;
+  void remove(const BlockId &blockId) override;
+  void remove(cpputils::unique_ref<Block> node) override;
   uint64_t numBlocks() const override;
   uint64_t estimateNumFreeBytes() const override;
   uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
+  void forEachBlock(std::function<void (const BlockId &)> callback) const override;
 
 private:
   cpputils::unique_ref<BlockStore> _baseBlockStore;
-  parallelaccessstore::ParallelAccessStore<Block, BlockRef, Key> _parallelAccessStore;
+  parallelaccessstore::ParallelAccessStore<Block, BlockRef, BlockId> _parallelAccessStore;
 
   DISALLOW_COPY_AND_ASSIGN(ParallelAccessBlockStore);
 };