Merged blockstore

.gitignore

@@ -1,3 +1,4 @@
 /build
 /cmake
 /.idea
+

.travis.yml

@@ -35,5 +35,6 @@ script:
 - ./test/cpp-utils/cpp-utils-test
 - ./run_with_fuse.sh ./test/fspp/fspp-test
 - ./test/parallelaccessstore/parallelaccessstore-test
+- ./test/blockstore/blockstore-test
 after_script:
 - rm run_with_fuse.sh

src/CMakeLists.txt

@@ -1,4 +1,5 @@
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
 add_subdirectory(cpp-utils)
-add_subdirectory(fspp)
+add_subdirectory(fspp)
+add_subdirectory(blockstore)

src/blockstore/CMakeLists.txt

@@ -0,0 +1,41 @@
+project (blockstore)
+
+set(SOURCES
+  utils/Key.cpp
+  utils/BlockStoreUtils.cpp
+  utils/FileDoesntExistException.cpp
+  interface/helpers/BlockStoreWithRandomKeys.cpp
+  implementations/testfake/FakeBlockStore.cpp
+  implementations/testfake/FakeBlock.cpp
+  implementations/inmemory/InMemoryBlock.cpp
+  implementations/inmemory/InMemoryBlockStore.cpp
+  implementations/parallelaccess/ParallelAccessBlockStore.cpp
+  implementations/parallelaccess/BlockRef.cpp
+  implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp
+  implementations/compressing/CompressingBlockStore.cpp
+  implementations/compressing/CompressedBlock.cpp
+  implementations/compressing/compressors/RunLengthEncoding.cpp
+  implementations/compressing/compressors/Gzip.cpp
+  implementations/encrypted/EncryptedBlockStore.cpp
+  implementations/encrypted/EncryptedBlock.cpp
+  implementations/ondisk/OnDiskBlockStore.cpp
+  implementations/ondisk/OnDiskBlock.cpp
+  implementations/caching/CachingBlockStore.cpp
+  implementations/caching/cache/PeriodicTask.cpp
+  implementations/caching/cache/CacheEntry.cpp
+  implementations/caching/cache/Cache.cpp
+  implementations/caching/cache/QueueMap.cpp
+  implementations/caching/CachedBlock.cpp
+  implementations/caching/NewBlock.cpp
+)
+
+add_library(${PROJECT_NAME} STATIC ${SOURCES})
+
+# This is needed by boost thread
+if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+  target_link_libraries(${PROJECT_NAME} PRIVATE rt)
+endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+
+target_add_boost(${PROJECT_NAME} filesystem system thread)
+target_enable_style_warnings(${PROJECT_NAME})
+target_activate_cpp14(${PROJECT_NAME})

src/blockstore/implementations/caching/CachedBlock.cpp

@@ -0,0 +1,46 @@
+#include "CachedBlock.h"
+#include "CachingBlockStore.h"
+
+using cpputils::unique_ref;
+
+namespace blockstore {
+namespace caching {
+
+CachedBlock::CachedBlock(unique_ref<Block> baseBlock, CachingBlockStore *blockStore)
+    :Block(baseBlock->key()),
+     _blockStore(blockStore),
+     _baseBlock(std::move(baseBlock)) {
+}
+
+CachedBlock::~CachedBlock() {
+  if (_baseBlock.get() != nullptr) {
+    _blockStore->release(std::move(_baseBlock));
+  }
+}
+
+const void *CachedBlock::data() const {
+  return _baseBlock->data();
+}
+
+void CachedBlock::write(const void *source, uint64_t offset, uint64_t size) {
+  return _baseBlock->write(source, offset, size);
+}
+
+void CachedBlock::flush() {
+  return _baseBlock->flush();
+}
+
+size_t CachedBlock::size() const {
+  return _baseBlock->size();
+}
+
+void CachedBlock::resize(size_t newSize) {
+    return _baseBlock->resize(newSize);
+}
+
+unique_ref<Block> CachedBlock::releaseBlock() {
+  return std::move(_baseBlock);
+}
+
+}
+}

src/blockstore/implementations/caching/CachedBlock.h

@@ -0,0 +1,39 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
+
+#include "../../interface/Block.h"
+
+#include <messmer/cpp-utils/pointer/unique_ref.h>
+
+namespace blockstore {
+namespace caching {
+class CachingBlockStore;
+
+class CachedBlock final: public Block {
+public:
+  //TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
+  CachedBlock(cpputils::unique_ref<Block> baseBlock, CachingBlockStore *blockStore);
+  ~CachedBlock();
+
+  const void *data() const override;
+  void write(const void *source, uint64_t offset, uint64_t size) override;
+  void flush() override;
+
+  size_t size() const override;
+
+  void resize(size_t newSize) override;
+
+  cpputils::unique_ref<Block> releaseBlock();
+
+private:
+  CachingBlockStore *_blockStore;
+  cpputils::unique_ref<Block> _baseBlock;
+
+  DISALLOW_COPY_AND_ASSIGN(CachedBlock);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/caching/CachingBlockStore.cpp

@@ -0,0 +1,92 @@
+#include "CachedBlock.h"
+#include "NewBlock.h"
+#include "CachingBlockStore.h"
+#include "../../interface/Block.h"
+
+#include <algorithm>
+#include <messmer/cpp-utils/pointer/cast.h>
+#include <messmer/cpp-utils/assert/assert.h>
+
+using cpputils::dynamic_pointer_move;
+using cpputils::Data;
+using boost::optional;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using boost::none;
+
+namespace blockstore {
+namespace caching {
+
+CachingBlockStore::CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore)
+  :_baseBlockStore(std::move(baseBlockStore)), _cache(), _numNewBlocks(0) {
+}
+
+Key CachingBlockStore::createKey() {
+  return _baseBlockStore->createKey();
+}
+
+optional<unique_ref<Block>> CachingBlockStore::tryCreate(const Key &key, Data data) {
+  ASSERT(_cache.pop(key) == none, "Key already exists in cache");
+  //TODO Shouldn't we return boost::none if the key already exists?
+  //TODO Key can also already exist but not be in the cache right now.
+  ++_numNewBlocks;
+  return unique_ref<Block>(make_unique_ref<CachedBlock>(make_unique_ref<NewBlock>(key, std::move(data), this), this));
+}
+
+optional<unique_ref<Block>> CachingBlockStore::load(const Key &key) {
+  optional<unique_ref<Block>> optBlock = _cache.pop(key);
+  //TODO an optional<> class with .getOrElse() would make this code simpler. boost::optional<>::value_or_eval didn't seem to work with unique_ptr members.
+  if (optBlock != none) {
+    return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*optBlock), this));
+  } else {
+    auto block = _baseBlockStore->load(key);
+    if (block == none) {
+      return none;
+    } else {
+      return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*block), this));
+    }
+  }
+}
+
+void CachingBlockStore::remove(cpputils::unique_ref<Block> block) {
+  auto cached_block = dynamic_pointer_move<CachedBlock>(block);
+  ASSERT(cached_block != none, "Passed block is not a CachedBlock");
+  auto baseBlock = (*cached_block)->releaseBlock();
+  auto baseNewBlock = dynamic_pointer_move<NewBlock>(baseBlock);
+  if (baseNewBlock != none) {
+	if(!(*baseNewBlock)->alreadyExistsInBaseStore()) {
+	  --_numNewBlocks;
+	}
+    (*baseNewBlock)->remove();
+  } else {
+    _baseBlockStore->remove(std::move(baseBlock));
+  }
+}
+
+uint64_t CachingBlockStore::numBlocks() const {
+  return _baseBlockStore->numBlocks() + _numNewBlocks;
+}
+
+void CachingBlockStore::release(unique_ref<Block> block) {
+  Key key = block->key();
+  _cache.push(key, std::move(block));
+}
+
+optional<unique_ref<Block>> CachingBlockStore::tryCreateInBaseStore(const Key &key, Data data) {
+  auto block = _baseBlockStore->tryCreate(key, std::move(data));
+  if (block != none) {
+	--_numNewBlocks;
+  }
+  return block;
+}
+
+void CachingBlockStore::removeFromBaseStore(cpputils::unique_ref<Block> block) {
+  _baseBlockStore->remove(std::move(block));
+}
+
+void CachingBlockStore::flush() {
+  _cache.flush();
+}
+
+}
+}

src/blockstore/implementations/caching/CachingBlockStore.h

@@ -0,0 +1,40 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
+
+#include "cache/Cache.h"
+#include "../../interface/BlockStore.h"
+
+namespace blockstore {
+namespace caching {
+
+//TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel)
+class CachingBlockStore final: public BlockStore {
+public:
+  explicit CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
+
+  Key createKey() override;
+  boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
+  boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
+  void remove(cpputils::unique_ref<Block> block) override;
+  uint64_t numBlocks() const override;
+
+  void release(cpputils::unique_ref<Block> block);
+
+  boost::optional<cpputils::unique_ref<Block>> tryCreateInBaseStore(const Key &key, cpputils::Data data);
+  void removeFromBaseStore(cpputils::unique_ref<Block> block);
+
+  void flush();
+
+private:
+  cpputils::unique_ref<BlockStore> _baseBlockStore;
+  Cache<Key, cpputils::unique_ref<Block>, 1000> _cache;
+  uint32_t _numNewBlocks;
+
+  DISALLOW_COPY_AND_ASSIGN(CachingBlockStore);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/caching/NewBlock.cpp

@@ -0,0 +1,75 @@
+#include "NewBlock.h"
+#include "CachingBlockStore.h"
+#include <messmer/cpp-utils/assert/assert.h>
+#include <messmer/cpp-utils/data/DataUtils.h>
+
+using cpputils::Data;
+using boost::none;
+
+namespace blockstore {
+namespace caching {
+
+NewBlock::NewBlock(const Key &key, Data data, CachingBlockStore *blockStore)
+    :Block(key),
+     _blockStore(blockStore),
+     _data(std::move(data)),
+     _baseBlock(none),
+     _dataChanged(true) {
+}
+
+NewBlock::~NewBlock() {
+  writeToBaseBlockIfChanged();
+}
+
+const void *NewBlock::data() const {
+  return _data.data();
+}
+
+void NewBlock::write(const void *source, uint64_t offset, uint64_t size) {
+  ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area");
+  std::memcpy((uint8_t*)_data.data()+offset, source, size);
+  _dataChanged = true;
+}
+
+void NewBlock::writeToBaseBlockIfChanged() {
+  if (_dataChanged) {
+    if (_baseBlock == none) {
+      //TODO _data.copy() necessary?
+      auto newBase = _blockStore->tryCreateInBaseStore(key(), _data.copy());
+      ASSERT(newBase != boost::none, "Couldn't create base block"); //TODO What if tryCreate fails due to a duplicate key? We should ensure we don't use duplicate keys.
+      _baseBlock = std::move(*newBase);
+    } else {
+        (*_baseBlock)->write(_data.data(), 0, _data.size());
+    }
+	_dataChanged = false;
+  }
+}
+
+void NewBlock::remove() {
+  if (_baseBlock != none) {
+	_blockStore->removeFromBaseStore(std::move(*_baseBlock));
+  }
+  _dataChanged = false;
+}
+
+void NewBlock::flush() {
+  writeToBaseBlockIfChanged();
+  ASSERT(_baseBlock != none, "At this point, the base block should already have been created but wasn't");
+  (*_baseBlock)->flush();
+}
+
+size_t NewBlock::size() const {
+  return _data.size();
+}
+
+void NewBlock::resize(size_t newSize) {
+    _data = cpputils::DataUtils::resize(std::move(_data), newSize);
+    _dataChanged = true;
+}
+
+bool NewBlock::alreadyExistsInBaseStore() const {
+  return _baseBlock != none;
+}
+
+}
+}

src/blockstore/implementations/caching/NewBlock.h

@@ -0,0 +1,52 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
+
+#include "../../interface/BlockStore.h"
+#include <messmer/cpp-utils/data/Data.h>
+
+#include "messmer/cpp-utils/macros.h"
+#include <memory>
+
+namespace blockstore {
+namespace caching {
+class CachingBlockStore;
+
+//TODO Does it make sense to write a general DataBackedBlock that just stores a Data object and maps the block operations to it?
+//     Can we reuse that object somewhere else?
+//     Maybe a second abstract class for BlockRefBackedBlock?
+
+// This is a block that was created in CachingBlockStore, but doesn't exist in the base block store yet.
+// It only exists in the cache and it is created in the base block store when destructed.
+class NewBlock final: public Block {
+public:
+  NewBlock(const Key &key, cpputils::Data data, CachingBlockStore *blockStore);
+  ~NewBlock();
+
+  const void *data() const override;
+  void write(const void *source, uint64_t offset, uint64_t size) override;
+  void flush() override;
+
+  size_t size() const override;
+
+  void resize(size_t newSize) override;
+
+  void remove();
+
+  bool alreadyExistsInBaseStore() const;
+
+private:
+  CachingBlockStore *_blockStore;
+  cpputils::Data _data;
+  boost::optional<cpputils::unique_ref<Block>> _baseBlock;
+  bool _dataChanged;
+
+  void writeToBaseBlockIfChanged();
+
+  DISALLOW_COPY_AND_ASSIGN(NewBlock);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/caching/cache/Cache.cpp

@@ -0,0 +1 @@
+#include "Cache.h"

src/blockstore/implementations/caching/cache/Cache.h

@@ -0,0 +1,178 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHE_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHE_H_
+
+#include "CacheEntry.h"
+#include "QueueMap.h"
+#include "PeriodicTask.h"
+#include <memory>
+#include <boost/optional.hpp>
+#include <future>
+#include <messmer/cpp-utils/assert/assert.h>
+#include <messmer/cpp-utils/lock/MutexPoolLock.h>
+#include <messmer/cpp-utils/pointer/gcc_4_8_compatibility.h>
+
+namespace blockstore {
+namespace caching {
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+class Cache final {
+public:
+  //TODO Current MAX_LIFETIME_SEC only considers time since the element was last pushed to the Cache. Also insert a real MAX_LIFETIME_SEC that forces resync of entries that have been pushed/popped often (e.g. the root blob)
+  //TODO Experiment with good values
+  static constexpr double PURGE_LIFETIME_SEC = 0.5; //When an entry has this age, it will be purged from the cache
+  static constexpr double PURGE_INTERVAL = 0.5; // With this interval, we check for entries to purge
+  static constexpr double MAX_LIFETIME_SEC = PURGE_LIFETIME_SEC + PURGE_INTERVAL; // This is the oldest age an entry can reach (given purging works in an ideal world, i.e. with the ideal interval and in zero time)
+
+  Cache();
+  ~Cache();
+
+  uint32_t size() const;
+
+  void push(const Key &key, Value value);
+  boost::optional<Value> pop(const Key &key);
+
+  void flush();
+
+private:
+  void _makeSpaceForEntry(std::unique_lock<std::mutex> *lock);
+  void _deleteEntry(std::unique_lock<std::mutex> *lock);
+  void _deleteOldEntriesParallel();
+  void _deleteAllEntriesParallel();
+  void _deleteMatchingEntriesAtBeginningParallel(std::function<bool (const CacheEntry<Key, Value> &)> matches);
+  void _deleteMatchingEntriesAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches);
+  bool _deleteMatchingEntryAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches);
+
+  mutable std::mutex _mutex;
+  cpputils::LockPool<Key> _currentlyFlushingEntries;
+  QueueMap<Key, CacheEntry<Key, Value>> _cachedBlocks;
+  std::unique_ptr<PeriodicTask> _timeoutFlusher;
+
+  DISALLOW_COPY_AND_ASSIGN(Cache);
+};
+
+template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Key, Value, MAX_ENTRIES>::PURGE_LIFETIME_SEC;
+template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Key, Value, MAX_ENTRIES>::PURGE_INTERVAL;
+template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Key, Value, MAX_ENTRIES>::MAX_LIFETIME_SEC;
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+Cache<Key, Value, MAX_ENTRIES>::Cache(): _mutex(), _currentlyFlushingEntries(), _cachedBlocks(), _timeoutFlusher(nullptr) {
+  //Don't initialize timeoutFlusher in the initializer list,
+  //because it then might already call Cache::popOldEntries() before Cache is done constructing.
+  _timeoutFlusher = std::make_unique<PeriodicTask>(std::bind(&Cache::_deleteOldEntriesParallel, this), PURGE_INTERVAL);
+}
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+Cache<Key, Value, MAX_ENTRIES>::~Cache() {
+  _deleteAllEntriesParallel();
+  ASSERT(_cachedBlocks.size() == 0, "Error in _deleteAllEntriesParallel()");
+}
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+boost::optional<Value> Cache<Key, Value, MAX_ENTRIES>::pop(const Key &key) {
+  std::unique_lock<std::mutex> lock(_mutex);
+  cpputils::MutexPoolLock<Key> lockEntryFromBeingPopped(&_currentlyFlushingEntries, key, &lock);
+
+  auto found = _cachedBlocks.pop(key);
+  if (!found) {
+    return boost::none;
+  }
+  return found->releaseValue();
+}
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+void Cache<Key, Value, MAX_ENTRIES>::push(const Key &key, Value value) {
+  std::unique_lock<std::mutex> lock(_mutex);
+  ASSERT(_cachedBlocks.size() <= MAX_ENTRIES, "Cache too full");
+  _makeSpaceForEntry(&lock);
+  _cachedBlocks.push(key, CacheEntry<Key, Value>(std::move(value)));
+}
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+void Cache<Key, Value, MAX_ENTRIES>::_makeSpaceForEntry(std::unique_lock<std::mutex> *lock) {
+  // _deleteEntry releases the lock while the Value destructor is running.
+  // So we can destruct multiple entries in parallel and also call pop() or push() while doing so.
+  // However, if another thread calls push() before we get the lock back, the cache is full again.
+  // That's why we need the while() loop here.
+  while (_cachedBlocks.size() == MAX_ENTRIES) {
+    _deleteEntry(lock);
+  }
+  ASSERT(_cachedBlocks.size() < MAX_ENTRIES, "Removing entry from cache didn't work");
+};
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+void Cache<Key, Value, MAX_ENTRIES>::_deleteEntry(std::unique_lock<std::mutex> *lock) {
+  auto key = _cachedBlocks.peekKey();
+  ASSERT(key != boost::none, "There was no entry to delete");
+  cpputils::MutexPoolLock<Key> lockEntryFromBeingPopped(&_currentlyFlushingEntries, *key);
+  auto value = _cachedBlocks.pop();
+  // Call destructor outside of the unique_lock,
+  // i.e. pop() and push() can be called here, except for pop() on the element in _currentlyFlushingEntries
+  lock->unlock();
+  value = boost::none; // Call destructor
+  lock->lock();
+};
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+void Cache<Key, Value, MAX_ENTRIES>::_deleteAllEntriesParallel() {
+  return _deleteMatchingEntriesAtBeginningParallel([] (const CacheEntry<Key, Value> &) {
+      return true;
+  });
+}
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+void Cache<Key, Value, MAX_ENTRIES>::_deleteOldEntriesParallel() {
+  return _deleteMatchingEntriesAtBeginningParallel([] (const CacheEntry<Key, Value> &entry) {
+      return entry.ageSeconds() > PURGE_LIFETIME_SEC;
+  });
+}
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+void Cache<Key, Value, MAX_ENTRIES>::_deleteMatchingEntriesAtBeginningParallel(std::function<bool (const CacheEntry<Key, Value> &)> matches) {
+  // Twice the number of cores, so we use full CPU even if half the threads are doing I/O
+  unsigned int numThreads = 2 * std::max(1u, std::thread::hardware_concurrency());
+  std::vector<std::future<void>> waitHandles;
+  for (unsigned int i = 0; i < numThreads; ++i) {
+    waitHandles.push_back(std::async(std::launch::async, [this, matches] {
+      _deleteMatchingEntriesAtBeginning(matches);
+    }));
+  }
+  for (auto & waitHandle : waitHandles) {
+    waitHandle.wait();
+  }
+};
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+void Cache<Key, Value, MAX_ENTRIES>::_deleteMatchingEntriesAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches) {
+  while (_deleteMatchingEntryAtBeginning(matches)) {}
+}
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+bool Cache<Key, Value, MAX_ENTRIES>::_deleteMatchingEntryAtBeginning(std::function<bool (const CacheEntry<Key, Value> &)> matches) {
+  // This function can be called in parallel by multiple threads and will then cause the Value destructors
+  // to be called in parallel. The call to _deleteEntry() releases the lock while the Value destructor is running.
+  std::unique_lock<std::mutex> lock(_mutex);
+  if (_cachedBlocks.size() > 0 && matches(*_cachedBlocks.peek())) {
+    _deleteEntry(&lock);
+    return true;
+  } else {
+    return false;
+  }
+};
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+uint32_t Cache<Key, Value, MAX_ENTRIES>::size() const {
+  std::unique_lock<std::mutex> lock(_mutex);
+  return _cachedBlocks.size();
+};
+
+template<class Key, class Value, uint32_t MAX_ENTRIES>
+void Cache<Key, Value, MAX_ENTRIES>::flush() {
+  //TODO Test flush()
+  return _deleteAllEntriesParallel();
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/caching/cache/CacheEntry.cpp

@@ -0,0 +1 @@
+#include "CacheEntry.h"

src/blockstore/implementations/caching/cache/CacheEntry.h

@@ -0,0 +1,44 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHEENTRY_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHEENTRY_H_
+
+#include <ctime>
+#include <memory>
+#include <messmer/cpp-utils/macros.h>
+#include <boost/date_time/posix_time/posix_time_types.hpp>
+
+namespace blockstore {
+namespace caching {
+
+template<class Key, class Value>
+class CacheEntry final {
+public:
+  explicit CacheEntry(Value value): _lastAccess(currentTime()), _value(std::move(value)) {
+  }
+
+  CacheEntry(CacheEntry &&) = default;
+
+  double ageSeconds() const {
+    return ((double)(currentTime() - _lastAccess).total_nanoseconds()) / ((double)1000000000);
+  }
+
+  Value releaseValue() {
+    return std::move(_value);
+  }
+
+private:
+  boost::posix_time::ptime _lastAccess;
+  Value _value;
+
+  static boost::posix_time::ptime currentTime() {
+	return boost::posix_time::microsec_clock::local_time();
+  }
+
+  DISALLOW_COPY_AND_ASSIGN(CacheEntry);
+};
+
+}
+}
+
+
+#endif

src/blockstore/implementations/caching/cache/PeriodicTask.cpp

@@ -0,0 +1,27 @@
+#include "PeriodicTask.h"
+#include <messmer/cpp-utils/logging/logging.h>
+
+using std::function;
+using std::endl;
+using namespace cpputils::logging;
+
+namespace blockstore {
+namespace caching {
+
+PeriodicTask::PeriodicTask(function<void ()> task, double intervalSec) :
+        _task(task),
+        _interval((uint64_t)(UINT64_C(1000000000) * intervalSec)),
+        _thread(std::bind(&PeriodicTask::_loopIteration, this)) {
+    _thread.start();
+}
+
+bool PeriodicTask::_loopIteration() {
+  //Has to be boost::this_thread::sleep_for and not std::this_thread::sleep_for, because it has to be interruptible.
+  //LoopThread will interrupt this method if it has to be restarted.
+  boost::this_thread::sleep_for(_interval);
+  _task();
+  return true; // Run another iteration (don't terminate thread)
+}
+
+}
+}

src/blockstore/implementations/caching/cache/PeriodicTask.h

@@ -0,0 +1,32 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_PERIODICTASK_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_PERIODICTASK_H_
+
+#include <functional>
+#include <messmer/cpp-utils/thread/LoopThread.h>
+#include <boost/chrono.hpp>
+
+namespace blockstore {
+namespace caching {
+
+class PeriodicTask final {
+public:
+	PeriodicTask(std::function<void ()> task, double intervalSec);
+
+private:
+  bool _loopIteration();
+
+  std::function<void()> _task;
+  boost::chrono::nanoseconds _interval;
+
+  //This member has to be last, so the thread is destructed first. Otherwise the thread might access elements from a
+  //partly destructed PeriodicTask.
+  cpputils::LoopThread _thread;
+
+  DISALLOW_COPY_AND_ASSIGN(PeriodicTask);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/caching/cache/QueueMap.cpp

@@ -0,0 +1 @@
+#include "QueueMap.h"

src/blockstore/implementations/caching/cache/QueueMap.h

@@ -0,0 +1,121 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_QUEUEMAP_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_QUEUEMAP_H_
+
+#include <memory>
+#include <unordered_map>
+#include <cassert>
+#include <boost/optional.hpp>
+#include <messmer/cpp-utils/macros.h>
+#include <messmer/cpp-utils/assert/assert.h>
+
+namespace blockstore {
+namespace caching {
+
+//TODO FreeList for performance (malloc is expensive)
+//TODO Single linked list with pointer to last element (for insertion) should be enough for a queue. No double linked list needed.
+//     But then, popping arbitrary elements needs to be rewritten so that _removeFromQueue() is _removeSuccessorFromQueue()
+//     and the map doesn't store the element itself, but its predecessor. That is, popping might be a bit slower. Test with experiments!
+
+// A class that is a queue and a map at the same time. We could also see it as an addressable queue.
+template<class Key, class Value>
+class QueueMap final {
+public:
+  QueueMap(): _entries(), _sentinel(&_sentinel, &_sentinel) {
+  }
+  ~QueueMap() {
+    for (auto &entry : _entries) {
+      entry.second.release();
+    }
+  }
+
+  void push(const Key &key, Value value) {
+    auto newEntry = _entries.emplace(std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple(_sentinel.prev, &_sentinel));
+    if (!newEntry.second) {
+      throw std::logic_error("There is already an element with this key");
+    }
+    newEntry.first->second.init(&newEntry.first->first, std::move(value));
+    //The following is ok, because std::unordered_map never invalidates pointers to its entries
+    _sentinel.prev->next = &newEntry.first->second;
+    _sentinel.prev = &newEntry.first->second;
+  }
+
+  boost::optional<Value> pop(const Key &key) {
+    auto found = _entries.find(key);
+    if (found == _entries.end()) {
+      return boost::none;
+    }
+    _removeFromQueue(found->second);
+    auto value = found->second.release();
+    _entries.erase(found);
+    return std::move(value);
+  }
+
+  boost::optional<Value> pop() {
+    if(_sentinel.next == &_sentinel) {
+      return boost::none;
+    }
+    return pop(*_sentinel.next->key);
+  }
+
+  boost::optional<const Key &> peekKey() {
+    if(_sentinel.next == &_sentinel) {
+      return boost::none;
+    }
+    return *_sentinel.next->key;
+  }
+
+  boost::optional<const Value &> peek() {
+    if(_sentinel.next == &_sentinel) {
+      return boost::none;
+    }
+    return _sentinel.next->value();
+  }
+
+  uint32_t size() const {
+    return _entries.size();
+  }
+
+private:
+  class Entry final {
+  public:
+    Entry(Entry *prev_, Entry *next_): prev(prev_), next(next_), key(nullptr), __value() {
+    }
+    void init(const Key *key_, Value value_) {
+      key = key_;
+      new(__value) Value(std::move(value_));
+    }
+    Value release() {
+      Value value = std::move(*_value());
+      _value()->~Value();
+      return value;
+    }
+    const Value &value() {
+      return *_value();
+    }
+    Entry *prev;
+    Entry *next;
+    const Key *key;
+  private:
+    Value *_value() {
+      return reinterpret_cast<Value*>(__value);
+    }
+    alignas(Value) char __value[sizeof(Value)];
+    DISALLOW_COPY_AND_ASSIGN(Entry);
+  };
+
+  void _removeFromQueue(const Entry &entry) {
+    entry.prev->next = entry.next;
+    entry.next->prev = entry.prev;
+  }
+
+  std::unordered_map<Key, Entry> _entries;
+  Entry _sentinel;
+
+  DISALLOW_COPY_AND_ASSIGN(QueueMap);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/compressing/CompressedBlock.cpp

@@ -0,0 +1 @@
+#include "CompressedBlock.h"

src/blockstore/implementations/compressing/CompressedBlock.h

@@ -0,0 +1,127 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSEDBLOCK_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSEDBLOCK_H_
+
+#include "../../interface/Block.h"
+#include "../../interface/BlockStore.h"
+#include <messmer/cpp-utils/data/DataUtils.h>
+#include <messmer/cpp-utils/pointer/unique_ref.h>
+#include <mutex>
+
+namespace blockstore {
+class BlockStore;
+namespace compressing {
+template<class Compressor> class CompressingBlockStore;
+
+template<class Compressor>
+class CompressedBlock final: public Block {
+public:
+  static boost::optional<cpputils::unique_ref<CompressedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData);
+  static cpputils::unique_ref<CompressedBlock> Decompress(cpputils::unique_ref<Block> baseBlock);
+
+  CompressedBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data decompressedData);
+  ~CompressedBlock();
+
+  const void *data() const override;
+  void write(const void *source, uint64_t offset, uint64_t size) override;
+
+  void flush() override;
+
+  size_t size() const override;
+  void resize(size_t newSize) override;
+
+  cpputils::unique_ref<Block> releaseBaseBlock();
+
+private:
+  void _compressToBaseBlock();
+
+  cpputils::unique_ref<Block> _baseBlock;
+  cpputils::Data _decompressedData;
+  std::mutex _mutex;
+  bool _dataChanged;
+
+  DISALLOW_COPY_AND_ASSIGN(CompressedBlock);
+};
+
+template<class Compressor>
+boost::optional<cpputils::unique_ref<CompressedBlock<Compressor>>> CompressedBlock<Compressor>::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData) {
+  cpputils::Data compressed = Compressor::Compress(decompressedData);
+  auto baseBlock = baseBlockStore->tryCreate(key, std::move(compressed));
+  if (baseBlock == boost::none) {
+    //TODO Test this code branch
+    return boost::none;
+  }
+
+  return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(*baseBlock), std::move(decompressedData));
+}
+
+template<class Compressor>
+cpputils::unique_ref<CompressedBlock<Compressor>> CompressedBlock<Compressor>::Decompress(cpputils::unique_ref<Block> baseBlock) {
+  cpputils::Data decompressed = Compressor::Decompress((byte*)baseBlock->data(), baseBlock->size());
+  return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(baseBlock), std::move(decompressed));
+}
+
+template<class Compressor>
+CompressedBlock<Compressor>::CompressedBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data decompressedData)
+        : Block(baseBlock->key()),
+          _baseBlock(std::move(baseBlock)),
+          _decompressedData(std::move(decompressedData)),
+          _dataChanged(false) {
+}
+
+template<class Compressor>
+CompressedBlock<Compressor>::~CompressedBlock() {
+  std::unique_lock<std::mutex> lock(_mutex);
+  _compressToBaseBlock();
+}
+
+template<class Compressor>
+const void *CompressedBlock<Compressor>::data() const {
+  return _decompressedData.data();
+}
+
+template<class Compressor>
+void CompressedBlock<Compressor>::write(const void *source, uint64_t offset, uint64_t size) {
+  std::memcpy((uint8_t*)_decompressedData.dataOffset(offset), source, size);
+  _dataChanged = true;
+}
+
+template<class Compressor>
+void CompressedBlock<Compressor>::flush() {
+  std::unique_lock<std::mutex> lock(_mutex);
+  _compressToBaseBlock();
+  return _baseBlock->flush();
+}
+
+template<class Compressor>
+size_t CompressedBlock<Compressor>::size() const {
+  return _decompressedData.size();
+}
+
+template<class Compressor>
+void CompressedBlock<Compressor>::resize(size_t newSize) {
+  _decompressedData = cpputils::DataUtils::resize(std::move(_decompressedData), newSize);
+  _dataChanged = true;
+}
+
+template<class Compressor>
+cpputils::unique_ref<Block> CompressedBlock<Compressor>::releaseBaseBlock() {
+  std::unique_lock<std::mutex> lock(_mutex);
+  _compressToBaseBlock();
+  return std::move(_baseBlock);
+}
+
+template<class Compressor>
+void CompressedBlock<Compressor>::_compressToBaseBlock() {
+  if (_dataChanged) {
+    cpputils::Data compressed = Compressor::Compress(_decompressedData);
+    _baseBlock->resize(compressed.size());
+    _baseBlock->write(compressed.data(), 0, compressed.size());
+    _dataChanged = false;
+  }
+}
+
+}
+}
+
+#endif

src/blockstore/implementations/compressing/CompressingBlockStore.cpp

@@ -0,0 +1 @@
+#include "CompressingBlockStore.h"

src/blockstore/implementations/compressing/CompressingBlockStore.h

@@ -0,0 +1,77 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSINGBLOCKSTORE_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSINGBLOCKSTORE_H_
+
+#include "../../interface/BlockStore.h"
+#include "CompressedBlock.h"
+
+namespace blockstore {
+namespace compressing {
+
+template<class Compressor>
+class CompressingBlockStore final: public BlockStore {
+public:
+    CompressingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
+    ~CompressingBlockStore();
+
+    Key createKey() override;
+    boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
+    boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
+    void remove(cpputils::unique_ref<Block> block) override;
+    uint64_t numBlocks() const override;
+
+private:
+    cpputils::unique_ref<BlockStore> _baseBlockStore;
+
+    DISALLOW_COPY_AND_ASSIGN(CompressingBlockStore);
+};
+
+template<class Compressor>
+CompressingBlockStore<Compressor>::CompressingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore)
+        : _baseBlockStore(std::move(baseBlockStore)) {
+}
+
+template<class Compressor>
+CompressingBlockStore<Compressor>::~CompressingBlockStore() {
+}
+
+template<class Compressor>
+Key CompressingBlockStore<Compressor>::createKey() {
+    return _baseBlockStore->createKey();
+}
+
+template<class Compressor>
+boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::tryCreate(const Key &key, cpputils::Data data) {
+    auto result = CompressedBlock<Compressor>::TryCreateNew(_baseBlockStore.get(), key, std::move(data));
+    if (result == boost::none) {
+        return boost::none;
+    }
+    return cpputils::unique_ref<Block>(std::move(*result));
+}
+
+template<class Compressor>
+boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::load(const Key &key) {
+    auto loaded = _baseBlockStore->load(key);
+    if (loaded == boost::none) {
+        return boost::none;
+    }
+    return boost::optional<cpputils::unique_ref<Block>>(CompressedBlock<Compressor>::Decompress(std::move(*loaded)));
+}
+
+template<class Compressor>
+void CompressingBlockStore<Compressor>::remove(cpputils::unique_ref<Block> block) {
+    auto _block = cpputils::dynamic_pointer_move<CompressedBlock<Compressor>>(block);
+    ASSERT(_block != boost::none, "Wrong block type");
+    auto baseBlock = (*_block)->releaseBaseBlock();
+    return _baseBlockStore->remove(std::move(baseBlock));
+}
+
+template<class Compressor>
+uint64_t CompressingBlockStore<Compressor>::numBlocks() const {
+    return _baseBlockStore->numBlocks();
+}
+
+}
+}
+
+#endif

src/blockstore/implementations/compressing/compressors/Gzip.cpp

@@ -0,0 +1,29 @@
+#include "Gzip.h"
+#include <cryptopp/cryptopp/gzip.h>
+
+using cpputils::Data;
+
+namespace blockstore {
+    namespace compressing {
+
+        Data Gzip::Compress(const Data &data) {
+            CryptoPP::Gzip zipper;
+            zipper.Put((byte *) data.data(), data.size());
+            zipper.MessageEnd();
+            Data compressed(zipper.MaxRetrievable());
+            zipper.Get((byte *) compressed.data(), compressed.size());
+            return compressed;
+        }
+
+        Data Gzip::Decompress(const void *data, size_t size) {
+            //TODO Change interface to taking cpputils::Data objects (needs changing blockstore so we can read their "class Data", because this is called from CompressedBlock::Decompress()).
+            CryptoPP::Gunzip zipper;
+            zipper.Put((byte *) data, size);
+            zipper.MessageEnd();
+            Data decompressed(zipper.MaxRetrievable());
+            zipper.Get((byte *) decompressed.data(), decompressed.size());
+            return decompressed;
+        }
+
+    }
+}

src/blockstore/implementations/compressing/compressors/Gzip.h

@@ -0,0 +1,18 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_GZIP_H
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_GZIP_H
+
+#include <messmer/cpp-utils/data/Data.h>
+
+namespace blockstore {
+    namespace compressing {
+        class Gzip {
+        public:
+            static cpputils::Data Compress(const cpputils::Data &data);
+
+            static cpputils::Data Decompress(const void *data, size_t size);
+        };
+    }
+}
+
+#endif

src/blockstore/implementations/compressing/compressors/RunLengthEncoding.cpp

@@ -0,0 +1,139 @@
+#include "RunLengthEncoding.h"
+#include <sstream>
+#include <messmer/cpp-utils/assert/assert.h>
+
+using cpputils::Data;
+using std::string;
+using std::ostringstream;
+using std::istringstream;
+
+namespace blockstore {
+    namespace compressing {
+
+        // Alternatively store a run of arbitrary bytes and a run of identical bytes.
+        // Each run is preceded by its length. Length fields are uint16_t.
+        // Example: 2 - 5 - 8 - 10 - 3 - 0 - 2 - 0
+        // Length 2 arbitrary bytes (values: 5, 8), the next 10 bytes store "3" each,
+        // then 0 arbitrary bytes and 2x "0".
+
+        Data RunLengthEncoding::Compress(const Data &data) {
+            ostringstream compressed;
+            uint8_t *current = (uint8_t*)data.data();
+            uint8_t *end = (uint8_t*)data.data()+data.size();
+            while (current < end) {
+                _encodeArbitraryWords(&current, end, &compressed);
+                ASSERT(current <= end, "Overflow");
+                if (current == end) {
+                    break;
+                }
+                _encodeIdenticalWords(&current, end, &compressed);
+                ASSERT(current <= end, "Overflow");
+            }
+            return _extractData(&compressed);
+        }
+
+        void RunLengthEncoding::_encodeArbitraryWords(uint8_t **current, uint8_t* end, ostringstream *output) {
+            uint16_t size = _arbitraryRunLength(*current, end);
+            output->write((const char*)&size, sizeof(uint16_t));
+            output->write((const char*)*current, size);
+            *current += size;
+        }
+
+        uint16_t RunLengthEncoding::_arbitraryRunLength(uint8_t *start, uint8_t* end) {
+            // Each stopping of an arbitrary bytes run costs us 5 byte, because we have to store the length
+            // for the identical bytes run (2 byte), the identical byte itself (1 byte) and the length for the next arbitrary bytes run (2 byte).
+            // So to get an advantage from stopping an arbitrary bytes run, at least 6 bytes have to be identical.
+
+            // realEnd avoids an overflow of the 16bit counter
+            uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max());
+
+            // Count the number of identical bytes and return if it finds a run of more than 6 identical bytes.
+            uint8_t lastByte = *start + 1; // Something different from the first byte
+            uint8_t numIdenticalBytes = 1;
+            for(uint8_t *current = start; current != realEnd; ++current) {
+                if (*current == lastByte) {
+                    ++numIdenticalBytes;
+                    if (numIdenticalBytes == 6) {
+                        return current - start - 5; //-5, because the end pointer for the arbitrary byte run should point to the first identical byte, not the one before.
+                    }
+                } else {
+                    numIdenticalBytes = 1;
+                }
+                lastByte = *current;
+            }
+            //It wasn't worth stopping the arbitrary bytes run anywhere. The whole region should be an arbitrary run.
+            return realEnd-start;
+        }
+
+        void RunLengthEncoding::_encodeIdenticalWords(uint8_t **current, uint8_t* end, ostringstream *output) {
+            uint16_t size = _countIdenticalBytes(*current, end);
+            output->write((const char*)&size, sizeof(uint16_t));
+            output->write((const char*)*current, 1);
+            *current += size;
+        }
+
+        uint16_t RunLengthEncoding::_countIdenticalBytes(uint8_t *start, uint8_t *end) {
+            uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max()); // This prevents overflow of the 16bit counter
+            for (uint8_t *current = start+1; current != realEnd; ++current) {
+                if (*current != *start) {
+                    return current-start;
+                }
+            }
+            // All bytes have been identical
+            return realEnd - start;
+        }
+
+        Data RunLengthEncoding::_extractData(ostringstream *stream) {
+            string str = stream->str();
+            Data data(str.size());
+            std::memcpy(data.data(), str.c_str(), str.size());
+            return data;
+        }
+
+        Data RunLengthEncoding::Decompress(const void *data, size_t size) {
+            istringstream stream;
+            _parseData((uint8_t*)data, size, &stream);
+            ostringstream decompressed;
+            while(_hasData(&stream)) {
+                _decodeArbitraryWords(&stream, &decompressed);
+                if (!_hasData(&stream)) {
+                    break;
+                }
+                _decodeIdenticalWords(&stream, &decompressed);
+            }
+            return _extractData(&decompressed);
+        }
+
+        bool RunLengthEncoding::_hasData(istringstream *str) {
+            str->peek();
+            return !str->eof();
+        }
+
+        void RunLengthEncoding::_parseData(const uint8_t *data, size_t size, istringstream *result) {
+            result->str(string((const char*)data, size));
+        }
+
+        void RunLengthEncoding::_decodeArbitraryWords(istringstream *stream, ostringstream *decompressed) {
+            uint16_t size;
+            stream->read((char*)&size, sizeof(uint16_t));
+            ASSERT(stream->good(), "Premature end of stream");
+            Data run(size);
+            stream->read((char*)run.data(), size);
+            ASSERT(stream->good(), "Premature end of stream");
+            decompressed->write((const char*)run.data(), run.size());
+        }
+
+        void RunLengthEncoding::_decodeIdenticalWords(istringstream *stream, ostringstream *decompressed) {
+            uint16_t size;
+            stream->read((char*)&size, sizeof(uint16_t));
+            ASSERT(stream->good(), "Premature end of stream");
+            uint8_t value;
+            stream->read((char*)&value, 1);
+            ASSERT(stream->good(), "Premature end of stream");
+            Data run(size);
+            std::memset(run.data(), value, run.size());
+            decompressed->write((const char*)run.data(), run.size());
+        }
+
+    }
+}

src/blockstore/implementations/compressing/compressors/RunLengthEncoding.h

@@ -0,0 +1,29 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_RUNLENGTHENCODING_H
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_RUNLENGTHENCODING_H
+
+#include <messmer/cpp-utils/data/Data.h>
+
+namespace blockstore {
+    namespace compressing {
+        class RunLengthEncoding {
+        public:
+            static cpputils::Data Compress(const cpputils::Data &data);
+
+            static cpputils::Data Decompress(const void *data, size_t size);
+
+        private:
+            static void _encodeArbitraryWords(uint8_t **current, uint8_t* end, std::ostringstream *output);
+            static uint16_t _arbitraryRunLength(uint8_t *start, uint8_t* end);
+            static void _encodeIdenticalWords(uint8_t **current, uint8_t* end, std::ostringstream *output);
+            static uint16_t _countIdenticalBytes(uint8_t *start, uint8_t *end);
+            static bool _hasData(std::istringstream *stream);
+            static cpputils::Data _extractData(std::ostringstream *stream);
+            static void _parseData(const uint8_t *data, size_t size, std::istringstream *result);
+            static void _decodeArbitraryWords(std::istringstream *stream, std::ostringstream *decompressed);
+            static void _decodeIdenticalWords(std::istringstream *stream, std::ostringstream *decompressed);
+        };
+    }
+}
+
+#endif

src/blockstore/implementations/encrypted/EncryptedBlock.cpp

@@ -0,0 +1 @@
+#include "EncryptedBlock.h"

src/blockstore/implementations/encrypted/EncryptedBlock.h

@@ -0,0 +1,177 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
+
+#include "../../interface/Block.h"
+#include <messmer/cpp-utils/data/Data.h>
+#include "../../interface/BlockStore.h"
+
+#include "messmer/cpp-utils/macros.h"
+#include <memory>
+#include <iostream>
+#include <boost/optional.hpp>
+#include <messmer/cpp-utils/crypto/symmetric/Cipher.h>
+#include <messmer/cpp-utils/assert/assert.h>
+#include <messmer/cpp-utils/data/DataUtils.h>
+#include <mutex>
+#include <messmer/cpp-utils/logging/logging.h>
+
+namespace blockstore {
+namespace encrypted {
+template<class Cipher> class EncryptedBlockStore;
+
+//TODO Test EncryptedBlock
+
+//TODO Fix mutexes & locks (basically true for all blockstores)
+
+template<class Cipher>
+class EncryptedBlock final: public Block {
+public:
+  BOOST_CONCEPT_ASSERT((cpputils::CipherConcept<Cipher>));
+  static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey);
+  static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key);
+
+  //TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
+  EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key, cpputils::Data plaintextWithHeader);
+  ~EncryptedBlock();
+
+  const void *data() const override;
+  void write(const void *source, uint64_t offset, uint64_t count) override;
+  void flush() override;
+
+  size_t size() const override;
+  void resize(size_t newSize) override;
+
+  cpputils::unique_ref<Block> releaseBlock();
+
+private:
+  cpputils::unique_ref<Block> _baseBlock; // TODO Do I need the ciphertext block in memory or is the key enough?
+  cpputils::Data _plaintextWithHeader;
+  typename Cipher::EncryptionKey _encKey;
+  bool _dataChanged;
+
+  static constexpr unsigned int HEADER_LENGTH = Key::BINARY_LENGTH;
+
+  void _encryptToBaseBlock();
+  static cpputils::Data _prependKeyHeaderToData(const Key &key, cpputils::Data data);
+  static bool _keyHeaderIsCorrect(const Key &key, const cpputils::Data &data);
+
+  std::mutex _mutex;
+
+  DISALLOW_COPY_AND_ASSIGN(EncryptedBlock);
+};
+
+template<class Cipher>
+constexpr unsigned int EncryptedBlock<Cipher>::HEADER_LENGTH;
+
+
+template<class Cipher>
+boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey) {
+  cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, std::move(data));
+  cpputils::Data encrypted = Cipher::encrypt((byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), encKey);
+  auto baseBlock = baseBlockStore->tryCreate(key, std::move(encrypted));
+  if (baseBlock == boost::none) {
+    //TODO Test this code branch
+    return boost::none;
+  }
+
+  return cpputils::make_unique_ref<EncryptedBlock>(std::move(*baseBlock), encKey, std::move(plaintextWithHeader));
+}
+
+template<class Cipher>
+boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey) {
+  //TODO Change BlockStore so we can read their "class Data" objects instead of "void *data()", and then we can change the Cipher interface to take Data objects instead of "byte *" + size
+  boost::optional<cpputils::Data> plaintextWithHeader = Cipher::decrypt((byte*)baseBlock->data(), baseBlock->size(), encKey);
+  if(plaintextWithHeader == boost::none) {
+    //Decryption failed (e.g. an authenticated cipher detected modifications to the ciphertext)
+    cpputils::logging::LOG(cpputils::logging::WARN) << "Decrypting block " << baseBlock->key().ToString() << " failed. Was the block modified by an attacker?";
+    return boost::none;
+  }
+  if(!_keyHeaderIsCorrect(baseBlock->key(), *plaintextWithHeader)) {
+    //The stored key in the block data is incorrect - an attacker might have exchanged the contents with the encrypted data from a different block
+    cpputils::logging::LOG(cpputils::logging::WARN) << "Decrypting block " << baseBlock->key().ToString() << " failed due to invalid block key. Was the block modified by an attacker?";
+    return boost::none;
+  }
+  return cpputils::make_unique_ref<EncryptedBlock<Cipher>>(std::move(baseBlock), encKey, std::move(*plaintextWithHeader));
+}
+
+template<class Cipher>
+cpputils::Data EncryptedBlock<Cipher>::_prependKeyHeaderToData(const Key &key, cpputils::Data data) {
+  static_assert(HEADER_LENGTH >= Key::BINARY_LENGTH, "Key doesn't fit into the header");
+  cpputils::Data result(data.size() + HEADER_LENGTH);
+  std::memcpy(result.data(), key.data(), Key::BINARY_LENGTH);
+  std::memcpy((uint8_t*)result.data() + Key::BINARY_LENGTH, data.data(), data.size());
+  return result;
+}
+
+template<class Cipher>
+bool EncryptedBlock<Cipher>::_keyHeaderIsCorrect(const Key &key, const cpputils::Data &data) {
+  return 0 == std::memcmp(key.data(), data.data(), Key::BINARY_LENGTH);
+}
+
+template<class Cipher>
+EncryptedBlock<Cipher>::EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey, cpputils::Data plaintextWithHeader)
+    :Block(baseBlock->key()),
+   _baseBlock(std::move(baseBlock)),
+   _plaintextWithHeader(std::move(plaintextWithHeader)),
+   _encKey(encKey),
+   _dataChanged(false),
+   _mutex() {
+}
+
+template<class Cipher>
+EncryptedBlock<Cipher>::~EncryptedBlock() {
+  std::unique_lock<std::mutex> lock(_mutex);
+  _encryptToBaseBlock();
+}
+
+template<class Cipher>
+const void *EncryptedBlock<Cipher>::data() const {
+  return (uint8_t*)_plaintextWithHeader.data() + HEADER_LENGTH;
+}
+
+template<class Cipher>
+void EncryptedBlock<Cipher>::write(const void *source, uint64_t offset, uint64_t count) {
+  ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition
+  std::memcpy((uint8_t*)_plaintextWithHeader.data()+HEADER_LENGTH+offset, source, count);
+  _dataChanged = true;
+}
+
+template<class Cipher>
+void EncryptedBlock<Cipher>::flush() {
+  std::unique_lock<std::mutex> lock(_mutex);
+  _encryptToBaseBlock();
+  return _baseBlock->flush();
+}
+
+template<class Cipher>
+size_t EncryptedBlock<Cipher>::size() const {
+  return _plaintextWithHeader.size() - HEADER_LENGTH;
+}
+
+template<class Cipher>
+void EncryptedBlock<Cipher>::resize(size_t newSize) {
+  _plaintextWithHeader = cpputils::DataUtils::resize(std::move(_plaintextWithHeader), newSize + HEADER_LENGTH);
+  _dataChanged = true;
+}
+
+template<class Cipher>
+void EncryptedBlock<Cipher>::_encryptToBaseBlock() {
+  if (_dataChanged) {
+    cpputils::Data encrypted = Cipher::encrypt((byte*)_plaintextWithHeader.data(), _plaintextWithHeader.size(), _encKey);
+    _baseBlock->write(encrypted.data(), 0, encrypted.size());
+    _dataChanged = false;
+  }
+}
+
+template<class Cipher>
+cpputils::unique_ref<Block> EncryptedBlock<Cipher>::releaseBlock() {
+  std::unique_lock<std::mutex> lock(_mutex);
+  _encryptToBaseBlock();
+  return std::move(_baseBlock);
+}
+
+}
+}
+
+#endif

src/blockstore/implementations/encrypted/EncryptedBlockStore.cpp

@@ -0,0 +1 @@
+#include "EncryptedBlockStore.h"

src/blockstore/implementations/encrypted/EncryptedBlockStore.h

@@ -0,0 +1,90 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
+
+#include "../../interface/BlockStore.h"
+#include <messmer/cpp-utils/macros.h>
+#include <messmer/cpp-utils/pointer/cast.h>
+#include "EncryptedBlock.h"
+#include <iostream>
+
+namespace blockstore {
+namespace encrypted {
+
+template<class Cipher>
+class EncryptedBlockStore final: public BlockStore {
+public:
+  EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey);
+
+  //TODO Are createKey() tests included in generic BlockStoreTest? If not, add it!
+  Key createKey() override;
+  boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
+  boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
+  void remove(cpputils::unique_ref<Block> block) override;
+  uint64_t numBlocks() const override;
+
+  //This function should only be used by test cases
+  void __setKey(const typename Cipher::EncryptionKey &encKey);
+
+private:
+  cpputils::unique_ref<BlockStore> _baseBlockStore;
+  typename Cipher::EncryptionKey _encKey;
+
+  DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore);
+};
+
+
+
+template<class Cipher>
+EncryptedBlockStore<Cipher>::EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
+ : _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
+}
+
+template<class Cipher>
+Key EncryptedBlockStore<Cipher>::createKey() {
+  return _baseBlockStore->createKey();
+}
+
+template<class Cipher>
+boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::tryCreate(const Key &key, cpputils::Data data) {
+  //TODO Test that this returns boost::none when base blockstore returns nullptr  (for all pass-through-blockstores)
+  //TODO Easier implementation? This is only so complicated because of the case EncryptedBlock -> Block
+  auto result = EncryptedBlock<Cipher>::TryCreateNew(_baseBlockStore.get(), key, std::move(data), _encKey);
+  if (result == boost::none) {
+    return boost::none;
+  }
+  return cpputils::unique_ref<Block>(std::move(*result));
+}
+
+template<class Cipher>
+boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::load(const Key &key) {
+  auto block = _baseBlockStore->load(key);
+  if (block == boost::none) {
+    //TODO Test this path (for all pass-through-blockstores)
+    return boost::none;
+  }
+  return boost::optional<cpputils::unique_ref<Block>>(EncryptedBlock<Cipher>::TryDecrypt(std::move(*block), _encKey));
+}
+
+template<class Cipher>
+void EncryptedBlockStore<Cipher>::remove(cpputils::unique_ref<Block> block) {
+  auto encryptedBlock = cpputils::dynamic_pointer_move<EncryptedBlock<Cipher>>(block);
+  ASSERT(encryptedBlock != boost::none, "Block is not an EncryptedBlock");
+  auto baseBlock = (*encryptedBlock)->releaseBlock();
+  return _baseBlockStore->remove(std::move(baseBlock));
+}
+
+template<class Cipher>
+uint64_t EncryptedBlockStore<Cipher>::numBlocks() const {
+  return _baseBlockStore->numBlocks();
+}
+
+template<class Cipher>
+void EncryptedBlockStore<Cipher>::__setKey(const typename Cipher::EncryptionKey &encKey) {
+  _encKey = encKey;
+}
+
+}
+}
+
+#endif

src/blockstore/implementations/inmemory/InMemoryBlock.cpp

@@ -0,0 +1,50 @@
+#include "InMemoryBlock.h"
+#include "InMemoryBlockStore.h"
+#include <cstring>
+#include <messmer/cpp-utils/data/DataUtils.h>
+#include <messmer/cpp-utils/assert/assert.h>
+
+using std::make_shared;
+using std::istream;
+using std::ostream;
+using std::ifstream;
+using std::ofstream;
+using std::ios;
+using cpputils::Data;
+
+namespace blockstore {
+namespace inmemory {
+
+InMemoryBlock::InMemoryBlock(const Key &key, Data data)
+ : Block(key), _data(make_shared<Data>(std::move(data))) {
+}
+
+InMemoryBlock::InMemoryBlock(const InMemoryBlock &rhs)
+ : Block(rhs), _data(rhs._data) {
+}
+
+InMemoryBlock::~InMemoryBlock() {
+}
+
+const void *InMemoryBlock::data() const {
+  return _data->data();
+}
+
+void InMemoryBlock::write(const void *source, uint64_t offset, uint64_t size) {
+  ASSERT(offset <= _data->size() && offset + size <= _data->size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
+  std::memcpy((uint8_t*)_data->data()+offset, source, size);
+}
+
+size_t InMemoryBlock::size() const {
+  return _data->size();
+}
+
+void InMemoryBlock::resize(size_t newSize) {
+    *_data = cpputils::DataUtils::resize(std::move(*_data), newSize);
+}
+
+void InMemoryBlock::flush() {
+}
+
+}
+}

src/blockstore/implementations/inmemory/InMemoryBlock.h

@@ -0,0 +1,33 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
+
+#include "../../interface/Block.h"
+#include <messmer/cpp-utils/data/Data.h>
+
+namespace blockstore {
+namespace inmemory {
+class InMemoryBlockStore;
+
+class InMemoryBlock final: public Block {
+public:
+  InMemoryBlock(const Key &key, cpputils::Data size);
+  InMemoryBlock(const InMemoryBlock &rhs);
+  ~InMemoryBlock();
+
+  const void *data() const override;
+  void write(const void *source, uint64_t offset, uint64_t size) override;
+
+  void flush() override;
+
+  size_t size() const override;
+  void resize(size_t newSize) override;
+
+private:
+  std::shared_ptr<cpputils::Data> _data;
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp

@@ -0,0 +1,56 @@
+#include "InMemoryBlock.h"
+#include "InMemoryBlockStore.h"
+#include <memory>
+#include <messmer/cpp-utils/assert/assert.h>
+
+using std::make_unique;
+using std::string;
+using std::mutex;
+using std::lock_guard;
+using std::piecewise_construct;
+using std::make_tuple;
+using cpputils::Data;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using boost::optional;
+using boost::none;
+
+namespace blockstore {
+namespace inmemory {
+
+InMemoryBlockStore::InMemoryBlockStore()
+ : _blocks() {}
+
+optional<unique_ref<Block>> InMemoryBlockStore::tryCreate(const Key &key, Data data) {
+  auto insert_result = _blocks.emplace(piecewise_construct, make_tuple(key.ToString()), make_tuple(key, std::move(data)));
+
+  if (!insert_result.second) {
+    return none;
+  }
+
+  //Return a pointer to the stored InMemoryBlock
+  return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(insert_result.first->second));
+}
+
+optional<unique_ref<Block>> InMemoryBlockStore::load(const Key &key) {
+  //Return a pointer to the stored InMemoryBlock
+  try {
+    return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(_blocks.at(key.ToString())));
+  } catch (const std::out_of_range &e) {
+    return none;
+  }
+}
+
+void InMemoryBlockStore::remove(unique_ref<Block> block) {
+  Key key = block->key();
+  cpputils::destruct(std::move(block));
+  int numRemoved = _blocks.erase(key.ToString());
+  ASSERT(1==numRemoved, "Didn't find block to remove");
+}
+
+uint64_t InMemoryBlockStore::numBlocks() const {
+  return _blocks.size();
+}
+
+}
+}

src/blockstore/implementations/inmemory/InMemoryBlockStore.h

@@ -0,0 +1,33 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
+
+#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
+#include <messmer/cpp-utils/macros.h>
+
+#include <mutex>
+#include <map>
+
+namespace blockstore {
+namespace inmemory {
+class InMemoryBlock;
+
+class InMemoryBlockStore final: public BlockStoreWithRandomKeys {
+public:
+  InMemoryBlockStore();
+
+  boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
+  boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
+  void remove(cpputils::unique_ref<Block> block) override;
+  uint64_t numBlocks() const override;
+
+private:
+  std::map<std::string, InMemoryBlock> _blocks;
+
+  DISALLOW_COPY_AND_ASSIGN(InMemoryBlockStore);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/ondisk/OnDiskBlock.cpp

@@ -0,0 +1,108 @@
+#include <cstring>
+#include <fstream>
+#include <boost/filesystem.hpp>
+#include "OnDiskBlock.h"
+#include "OnDiskBlockStore.h"
+#include "../../utils/FileDoesntExistException.h"
+#include <messmer/cpp-utils/data/DataUtils.h>
+#include <messmer/cpp-utils/assert/assert.h>
+
+using std::istream;
+using std::ostream;
+using std::ifstream;
+using std::ofstream;
+using std::ios;
+using cpputils::Data;
+using cpputils::make_unique_ref;
+using cpputils::unique_ref;
+using boost::optional;
+using boost::none;
+
+namespace bf = boost::filesystem;
+
+namespace blockstore {
+namespace ondisk {
+
+OnDiskBlock::OnDiskBlock(const Key &key, const bf::path &filepath, Data data)
+ : Block(key), _filepath(filepath), _data(std::move(data)), _dataChanged(false), _mutex() {
+}
+
+OnDiskBlock::~OnDiskBlock() {
+  flush();
+}
+
+const void *OnDiskBlock::data() const {
+  return _data.data();
+}
+
+void OnDiskBlock::write(const void *source, uint64_t offset, uint64_t size) {
+  ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
+  std::memcpy((uint8_t*)_data.data()+offset, source, size);
+  _dataChanged = true;
+}
+
+size_t OnDiskBlock::size() const {
+  return _data.size();
+}
+
+void OnDiskBlock::resize(size_t newSize) {
+  _data = cpputils::DataUtils::resize(std::move(_data), newSize);
+  _dataChanged = true;
+}
+
+optional<unique_ref<OnDiskBlock>> OnDiskBlock::LoadFromDisk(const bf::path &rootdir, const Key &key) {
+  auto filepath = rootdir / key.ToString();
+  try {
+    //If it isn't a file, Data::LoadFromFile() would usually also crash. We still need this extra check
+    //upfront, because Data::LoadFromFile() doesn't crash if we give it the path of a directory
+    //instead the path of a file.
+    //TODO Data::LoadFromFile now returns boost::optional. Do we then still need this?
+    if(!bf::is_regular_file(filepath)) {
+      return none;
+    }
+    boost::optional<Data> data = Data::LoadFromFile(filepath);
+    if (!data) {
+      return none;
+    }
+    return make_unique_ref<OnDiskBlock>(key, filepath, std::move(*data));
+  } catch (const FileDoesntExistException &e) {
+    return none;
+  }
+}
+
+optional<unique_ref<OnDiskBlock>> OnDiskBlock::CreateOnDisk(const bf::path &rootdir, const Key &key, Data data) {
+  auto filepath = rootdir / key.ToString();
+  if (bf::exists(filepath)) {
+    return none;
+  }
+
+  auto block = make_unique_ref<OnDiskBlock>(key, filepath, std::move(data));
+  block->_storeToDisk();
+  return std::move(block);
+}
+
+void OnDiskBlock::RemoveFromDisk(const bf::path &rootdir, const Key &key) {
+  auto filepath = rootdir / key.ToString();
+  ASSERT(bf::is_regular_file(filepath), "Block not found on disk");
+  bf::remove(filepath);
+}
+
+void OnDiskBlock::_fillDataWithZeroes() {
+  _data.FillWithZeroes();
+  _dataChanged = true;
+}
+
+void OnDiskBlock::_storeToDisk() const {
+  _data.StoreToFile(_filepath);
+}
+
+void OnDiskBlock::flush() {
+  std::unique_lock<std::mutex> lock(_mutex);
+  if (_dataChanged) {
+    _storeToDisk();
+    _dataChanged = false;
+  }
+}
+
+}
+}

src/blockstore/implementations/ondisk/OnDiskBlock.h

@@ -0,0 +1,50 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
+
+#include <boost/filesystem/path.hpp>
+#include "../../interface/Block.h"
+#include <messmer/cpp-utils/data/Data.h>
+#include <iostream>
+
+#include <messmer/cpp-utils/pointer/unique_ref.h>
+#include <mutex>
+
+namespace blockstore {
+namespace ondisk {
+class OnDiskBlockStore;
+
+class OnDiskBlock final: public Block {
+public:
+  OnDiskBlock(const Key &key, const boost::filesystem::path &filepath, cpputils::Data data);
+  ~OnDiskBlock();
+
+  static boost::optional<cpputils::unique_ref<OnDiskBlock>> LoadFromDisk(const boost::filesystem::path &rootdir, const Key &key);
+  static boost::optional<cpputils::unique_ref<OnDiskBlock>> CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data);
+  static void RemoveFromDisk(const boost::filesystem::path &rootdir, const Key &key);
+
+  const void *data() const override;
+  void write(const void *source, uint64_t offset, uint64_t size) override;
+
+  void flush() override;
+
+  size_t size() const override;
+  void resize(size_t newSize) override;
+
+private:
+  const boost::filesystem::path _filepath;
+  cpputils::Data _data;
+  bool _dataChanged;
+
+  void _fillDataWithZeroes();
+  void _storeToDisk() const;
+
+  std::mutex _mutex;
+
+  DISALLOW_COPY_AND_ASSIGN(OnDiskBlock);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp

@@ -0,0 +1,52 @@
+#include "OnDiskBlock.h"
+#include "OnDiskBlockStore.h"
+
+using std::string;
+using cpputils::Data;
+using cpputils::unique_ref;
+using boost::optional;
+using boost::none;
+
+namespace bf = boost::filesystem;
+
+namespace blockstore {
+namespace ondisk {
+
+OnDiskBlockStore::OnDiskBlockStore(const boost::filesystem::path &rootdir)
+ : _rootdir(rootdir) {
+  if (!bf::exists(rootdir)) {
+    throw std::runtime_error("Base directory not found");
+  }
+  if (!bf::is_directory(rootdir)) {
+      throw std::runtime_error("Base directory is not a directory");
+  }
+  //TODO Test for read access, write access, enter (x) access, and throw runtime_error in case
+}
+
+//TODO Do I have to lock tryCreate/remove and/or load? Or does ParallelAccessBlockStore take care of that?
+
+optional<unique_ref<Block>> OnDiskBlockStore::tryCreate(const Key &key, Data data) {
+  //TODO Easier implementation? This is only so complicated because of the cast OnDiskBlock -> Block
+  auto result = std::move(OnDiskBlock::CreateOnDisk(_rootdir, key, std::move(data)));
+  if (result == boost::none) {
+    return boost::none;
+  }
+  return unique_ref<Block>(std::move(*result));
+}
+
+optional<unique_ref<Block>> OnDiskBlockStore::load(const Key &key) {
+  return optional<unique_ref<Block>>(OnDiskBlock::LoadFromDisk(_rootdir, key));
+}
+
+void OnDiskBlockStore::remove(unique_ref<Block> block) {
+  Key key = block->key();
+  cpputils::destruct(std::move(block));
+  OnDiskBlock::RemoveFromDisk(_rootdir, key);
+}
+
+uint64_t OnDiskBlockStore::numBlocks() const {
+  return std::distance(bf::directory_iterator(_rootdir), bf::directory_iterator());
+}
+
+}
+}

src/blockstore/implementations/ondisk/OnDiskBlockStore.h

@@ -0,0 +1,31 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
+
+#include <boost/filesystem.hpp>
+#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
+
+#include "messmer/cpp-utils/macros.h"
+
+namespace blockstore {
+namespace ondisk {
+
+class OnDiskBlockStore final: public BlockStoreWithRandomKeys {
+public:
+  explicit OnDiskBlockStore(const boost::filesystem::path &rootdir);
+
+  boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
+  boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
+  void remove(cpputils::unique_ref<Block> block) override;
+  uint64_t numBlocks() const override;
+
+private:
+  const boost::filesystem::path _rootdir;
+
+  DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/parallelaccess/BlockRef.cpp

@@ -0,0 +1 @@
+#include "BlockRef.h"

src/blockstore/implementations/parallelaccess/BlockRef.h

@@ -0,0 +1,48 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_BLOCKREF_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_BLOCKREF_H_
+
+#include <messmer/parallelaccessstore/ParallelAccessStore.h>
+#include "../../interface/Block.h"
+#include "messmer/cpp-utils/macros.h"
+#include <memory>
+
+namespace blockstore {
+namespace parallelaccess {
+class ParallelAccessBlockStore;
+
+class BlockRef final: public Block, public parallelaccessstore::ParallelAccessStore<Block, BlockRef, Key>::ResourceRefBase {
+public:
+  //TODO Unneccessarily storing Key twice here (in parent class and in _baseBlock).
+  explicit BlockRef(Block *baseBlock): Block(baseBlock->key()), _baseBlock(baseBlock) {}
+
+  const void *data() const override {
+	return _baseBlock->data();
+  }
+
+  void write(const void *source, uint64_t offset, uint64_t size) override {
+	return _baseBlock->write(source, offset, size);
+  }
+
+  void flush() override {
+	return _baseBlock->flush();
+  }
+
+  size_t size() const override {
+	return _baseBlock->size();
+  }
+
+  void resize(size_t newSize) override {
+    return _baseBlock->resize(newSize);
+  }
+
+private:
+  Block *_baseBlock;
+
+  DISALLOW_COPY_AND_ASSIGN(BlockRef);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.cpp

@@ -0,0 +1,60 @@
+#include "BlockRef.h"
+#include "ParallelAccessBlockStore.h"
+#include "ParallelAccessBlockStoreAdapter.h"
+#include <cassert>
+#include <messmer/cpp-utils/pointer/cast.h>
+#include <messmer/cpp-utils/assert/assert.h>
+
+using std::string;
+using std::promise;
+using cpputils::dynamic_pointer_move;
+using cpputils::make_unique_ref;
+using boost::none;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using boost::optional;
+using boost::none;
+
+namespace blockstore {
+namespace parallelaccess {
+
+ParallelAccessBlockStore::ParallelAccessBlockStore(unique_ref<BlockStore> baseBlockStore)
+ : _baseBlockStore(std::move(baseBlockStore)), _parallelAccessStore(make_unique_ref<ParallelAccessBlockStoreAdapter>(_baseBlockStore.get())) {
+}
+
+Key ParallelAccessBlockStore::createKey() {
+  return _baseBlockStore->createKey();
+}
+
+optional<unique_ref<Block>> ParallelAccessBlockStore::tryCreate(const Key &key, cpputils::Data data) {
+  ASSERT(!_parallelAccessStore.isOpened(key), ("Key "+key.ToString()+"already exists").c_str());
+  auto block = _baseBlockStore->tryCreate(key, std::move(data));
+  if (block == none) {
+	//TODO Test this code branch
+	return none;
+  }
+  return unique_ref<Block>(_parallelAccessStore.add(key, std::move(*block)));
+}
+
+optional<unique_ref<Block>> ParallelAccessBlockStore::load(const Key &key) {
+  auto block = _parallelAccessStore.load(key);
+  if (block == none) {
+    return none;
+  }
+  return unique_ref<Block>(std::move(*block));
+}
+
+
+void ParallelAccessBlockStore::remove(unique_ref<Block> block) {
+  Key key = block->key();
+  auto block_ref = dynamic_pointer_move<BlockRef>(block);
+  ASSERT(block_ref != none, "Block is not a BlockRef");
+  return _parallelAccessStore.remove(key, std::move(*block_ref));
+}
+
+uint64_t ParallelAccessBlockStore::numBlocks() const {
+  return _baseBlockStore->numBlocks();
+}
+
+}
+}

src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h

@@ -0,0 +1,34 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTORE_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTORE_H_
+
+#include <messmer/parallelaccessstore/ParallelAccessStore.h>
+#include "BlockRef.h"
+#include "../../interface/BlockStore.h"
+#include <messmer/cpp-utils/pointer/unique_ref.h>
+
+namespace blockstore {
+namespace parallelaccess {
+
+//TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel)
+class ParallelAccessBlockStore final: public BlockStore {
+public:
+  explicit ParallelAccessBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
+
+  Key createKey() override;
+  boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
+  boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
+  void remove(cpputils::unique_ref<Block> block) override;
+  uint64_t numBlocks() const override;
+
+private:
+  cpputils::unique_ref<BlockStore> _baseBlockStore;
+  parallelaccessstore::ParallelAccessStore<Block, BlockRef, Key> _parallelAccessStore;
+
+  DISALLOW_COPY_AND_ASSIGN(ParallelAccessBlockStore);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp

@@ -0,0 +1 @@
+#include "ParallelAccessBlockStoreAdapter.h"

src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.h

@@ -0,0 +1,35 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTOREADAPTER_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTOREADAPTER_H_
+
+#include <messmer/cpp-utils/macros.h>
+#include <messmer/parallelaccessstore/ParallelAccessStore.h>
+#include "../../interface/BlockStore.h"
+
+namespace blockstore {
+namespace parallelaccess {
+
+class ParallelAccessBlockStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore<Block, Key> {
+public:
+  explicit ParallelAccessBlockStoreAdapter(BlockStore *baseBlockStore)
+    :_baseBlockStore(std::move(baseBlockStore)) {
+  }
+
+  boost::optional<cpputils::unique_ref<Block>> loadFromBaseStore(const Key &key) override {
+	return _baseBlockStore->load(key);
+  }
+
+  void removeFromBaseStore(cpputils::unique_ref<Block> block) override {
+	return _baseBlockStore->remove(std::move(block));
+  }
+
+private:
+  BlockStore *_baseBlockStore;
+
+  DISALLOW_COPY_AND_ASSIGN(ParallelAccessBlockStoreAdapter);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/testfake/FakeBlock.cpp

@@ -0,0 +1,54 @@
+#include "FakeBlock.h"
+#include "FakeBlockStore.h"
+#include <cstring>
+#include <messmer/cpp-utils/assert/assert.h>
+#include <messmer/cpp-utils/data/DataUtils.h>
+
+using std::shared_ptr;
+using std::istream;
+using std::ostream;
+using std::ifstream;
+using std::ofstream;
+using std::ios;
+using std::string;
+using cpputils::Data;
+
+namespace blockstore {
+namespace testfake {
+
+FakeBlock::FakeBlock(FakeBlockStore *store, const Key &key, shared_ptr<Data> data, bool dirty)
+ : Block(key), _store(store), _data(data), _dataChanged(dirty) {
+}
+
+FakeBlock::~FakeBlock() {
+  flush();
+}
+
+const void *FakeBlock::data() const {
+  return _data->data();
+}
+
+void FakeBlock::write(const void *source, uint64_t offset, uint64_t size) {
+  ASSERT(offset <= _data->size() && offset + size <= _data->size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
+  std::memcpy((uint8_t*)_data->data()+offset, source, size);
+  _dataChanged = true;
+}
+
+size_t FakeBlock::size() const {
+  return _data->size();
+}
+
+void FakeBlock::resize(size_t newSize) {
+  *_data = cpputils::DataUtils::resize(std::move(*_data), newSize);
+  _dataChanged = true;
+}
+
+void FakeBlock::flush() {
+  if(_dataChanged) {
+    _store->updateData(key(), *_data);
+    _dataChanged = false;
+  }
+}
+
+}
+}

src/blockstore/implementations/testfake/FakeBlock.h

@@ -0,0 +1,39 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCK_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCK_H_
+
+#include "../../interface/Block.h"
+#include <messmer/cpp-utils/data/Data.h>
+
+#include "messmer/cpp-utils/macros.h"
+
+namespace blockstore {
+namespace testfake {
+class FakeBlockStore;
+
+class FakeBlock final: public Block {
+public:
+  FakeBlock(FakeBlockStore *store, const Key &key, std::shared_ptr<cpputils::Data> data, bool dirty);
+  ~FakeBlock();
+
+  const void *data() const override;
+  void write(const void *source, uint64_t offset, uint64_t size) override;
+
+  void flush() override;
+
+  size_t size() const override;
+
+  void resize(size_t newSize) override;
+
+private:
+  FakeBlockStore *_store;
+  std::shared_ptr<cpputils::Data> _data;
+  bool _dataChanged;
+
+  DISALLOW_COPY_AND_ASSIGN(FakeBlock);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/testfake/FakeBlockStore.cpp

@@ -0,0 +1,80 @@
+#include "FakeBlock.h"
+#include "FakeBlockStore.h"
+#include <messmer/cpp-utils/assert/assert.h>
+
+using std::make_shared;
+using std::string;
+using std::mutex;
+using std::lock_guard;
+using cpputils::Data;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using boost::optional;
+using boost::none;
+
+namespace blockstore {
+namespace testfake {
+
+FakeBlockStore::FakeBlockStore()
+ : _blocks(), _used_dataregions_for_blocks(), _mutex() {}
+
+optional<unique_ref<Block>> FakeBlockStore::tryCreate(const Key &key, Data data) {
+  std::unique_lock<std::mutex> lock(_mutex);
+  auto insert_result = _blocks.emplace(key.ToString(), std::move(data));
+
+  if (!insert_result.second) {
+    return none;
+  }
+
+  //Return a copy of the stored data
+  return _load(key);
+}
+
+optional<unique_ref<Block>> FakeBlockStore::load(const Key &key) {
+  std::unique_lock<std::mutex> lock(_mutex);
+  return _load(key);
+}
+
+optional<unique_ref<Block>> FakeBlockStore::_load(const Key &key) {
+  //Return a copy of the stored data
+  string key_string = key.ToString();
+  try {
+    return makeFakeBlockFromData(key, _blocks.at(key_string), false);
+  } catch (const std::out_of_range &e) {
+    return none;
+  }
+}
+
+void FakeBlockStore::remove(unique_ref<Block> block) {
+  Key key = block->key();
+  cpputils::destruct(std::move(block));
+  std::unique_lock<std::mutex> lock(_mutex);
+  int numRemoved = _blocks.erase(key.ToString());
+  ASSERT(numRemoved == 1, "Block not found");
+}
+
+unique_ref<Block> FakeBlockStore::makeFakeBlockFromData(const Key &key, const Data &data, bool dirty) {
+  auto newdata = make_shared<Data>(data.copy());
+  _used_dataregions_for_blocks.push_back(newdata);
+  return make_unique_ref<FakeBlock>(this, key, newdata, dirty);
+}
+
+void FakeBlockStore::updateData(const Key &key, const Data &data) {
+  std::unique_lock<std::mutex> lock(_mutex);
+  auto found = _blocks.find(key.ToString());
+  if (found == _blocks.end()) {
+    auto insertResult = _blocks.emplace(key.ToString(), data.copy());
+    ASSERT(true == insertResult.second, "Inserting didn't work");
+    found = insertResult.first;
+  }
+  Data &stored_data = found->second;
+  stored_data = data.copy();
+}
+
+uint64_t FakeBlockStore::numBlocks() const {
+  std::unique_lock<std::mutex> lock(_mutex);
+  return _blocks.size();
+}
+
+}
+}

src/blockstore/implementations/testfake/FakeBlockStore.h

@@ -0,0 +1,62 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCKSTORE_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCKSTORE_H_
+
+#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
+#include <messmer/cpp-utils/data/Data.h>
+#include "messmer/cpp-utils/macros.h"
+
+#include <mutex>
+#include <map>
+
+namespace blockstore {
+namespace testfake {
+class FakeBlock;
+
+/**
+ * This blockstore is meant to be used for unit tests when the module under test needs a blockstore to work with.
+ * It basically is the same as the InMemoryBlockStore, but much less forgiving for programming mistakes.
+ *
+ * InMemoryBlockStore for example simply ignores flushing and gives you access to the same data region each time
+ * you request a block. This is very performant, but also forgiving to mistakes. Say you write over the boundaries
+ * of a block, then you wouldn't notice, since the next time you access the block, the overflow data is (probably)
+ * still there. Or say an application is relying on flushing the block store in the right moment. Since flushing
+ * is a no-op in InMemoryBlockStore, you wouldn't notice either.
+ *
+ * So this FakeBlockStore has a background copy of each block. When you request a block, you will get a copy of
+ * the data (instead of a direct pointer as InMemoryBlockStore does) and flushing will copy the data back to the
+ * background. This way, tests are more likely to fail if they use the blockstore wrongly.
+ */
+class FakeBlockStore final: public BlockStoreWithRandomKeys {
+public:
+  FakeBlockStore();
+
+  boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
+  boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
+  void remove(cpputils::unique_ref<Block> block) override;
+  uint64_t numBlocks() const override;
+
+  void updateData(const Key &key, const cpputils::Data &data);
+
+private:
+  std::map<std::string, cpputils::Data> _blocks;
+
+  //This vector keeps a handle of the data regions for all created FakeBlock objects.
+  //This way, it is ensured that no two created FakeBlock objects will work on the
+  //same data region. Without this, it could happen that a test case creates a FakeBlock,
+  //destructs it, creates another one, and the new one gets the same memory region.
+  //We want to avoid this for the reasons mentioned above (overflow data).
+  std::vector<std::shared_ptr<cpputils::Data>> _used_dataregions_for_blocks;
+
+  mutable std::mutex _mutex;
+
+  cpputils::unique_ref<Block> makeFakeBlockFromData(const Key &key, const cpputils::Data &data, bool dirty);
+  boost::optional<cpputils::unique_ref<Block>> _load(const Key &key);
+
+  DISALLOW_COPY_AND_ASSIGN(FakeBlockStore);
+};
+
+}
+}
+
+#endif

src/blockstore/interface/Block.h

@@ -0,0 +1,41 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_INTERFACE_BLOCK_H_
+#define MESSMER_BLOCKSTORE_INTERFACE_BLOCK_H_
+
+#include "../utils/Key.h"
+#include <cstring>
+
+namespace blockstore {
+
+//TODO Make Block non-virtual class that stores ptr to its blockstore and writes itself back to the blockstore who is offering a corresponding function.
+//     Then ondisk blockstore can be actually create the file on disk in blockstore::create() and cachingblockstore will delay that call to its base block store.
+
+class Block {
+public:
+  virtual ~Block() {}
+
+  virtual const void *data() const = 0;
+  virtual void write(const void *source, uint64_t offset, uint64_t size) = 0;
+
+  virtual void flush() = 0;
+
+  virtual size_t size() const = 0;
+
+  //TODO Test resize()
+  virtual void resize(size_t newSize) = 0;
+
+  const Key &key() const {
+    return _key;
+  }
+
+protected:
+  Block(const Key &key) : _key(key) {}
+
+private:
+  const Key _key;
+};
+
+}
+
+
+#endif

src/blockstore/interface/BlockStore.h

@@ -0,0 +1,39 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_INTERFACE_BLOCKSTORE_H_
+#define MESSMER_BLOCKSTORE_INTERFACE_BLOCKSTORE_H_
+
+#include "Block.h"
+#include <string>
+#include <boost/optional.hpp>
+#include <messmer/cpp-utils/pointer/unique_ref.h>
+#include <messmer/cpp-utils/data/Data.h>
+
+namespace blockstore {
+
+class BlockStore {
+public:
+  virtual ~BlockStore() {}
+
+  virtual Key createKey() = 0;
+  //Returns boost::none if key already exists
+  virtual boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) = 0;
+  //TODO Use boost::optional (if key doesn't exist)
+  // Return nullptr if block with this key doesn't exists
+  virtual boost::optional<cpputils::unique_ref<Block>> load(const Key &key) = 0;
+  virtual void remove(cpputils::unique_ref<Block> block) = 0;
+  virtual uint64_t numBlocks() const = 0;
+
+  cpputils::unique_ref<Block> create(const cpputils::Data &data) {
+    while(true) {
+      //TODO Copy (data.copy()) necessary?
+      auto block = tryCreate(createKey(), data.copy());
+      if (block != boost::none) {
+        return std::move(*block);
+      }
+    }
+  }
+};
+
+}
+
+#endif

src/blockstore/interface/helpers/BlockStoreWithRandomKeys.cpp

@@ -0,0 +1 @@
+#include "BlockStoreWithRandomKeys.h"

src/blockstore/interface/helpers/BlockStoreWithRandomKeys.h

@@ -0,0 +1,23 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_INTERFACE_HELPERS_BLOCKSTOREWITHRANDOMKEYS_H_
+#define MESSMER_BLOCKSTORE_INTERFACE_HELPERS_BLOCKSTOREWITHRANDOMKEYS_H_
+
+#include "../BlockStore.h"
+#include "../Block.h"
+#include <messmer/cpp-utils/random/Random.h>
+
+namespace blockstore {
+
+// This is an implementation helpers for BlockStores that use random block keys.
+// You should never give this static type to the client. The client should always
+// work with the BlockStore interface instead.
+class BlockStoreWithRandomKeys: public BlockStore {
+public:
+  Key createKey() final {
+    return cpputils::Random::PseudoRandom().getFixedSize<Key::BINARY_LENGTH>();
+  }
+};
+
+}
+
+#endif

src/blockstore/utils/BlockStoreUtils.cpp

@@ -0,0 +1,31 @@
+#include "../interface/BlockStore.h"
+#include "BlockStoreUtils.h"
+#include <messmer/cpp-utils/data/Data.h>
+#include <cassert>
+#include <messmer/cpp-utils/assert/assert.h>
+
+using cpputils::Data;
+using cpputils::unique_ref;
+
+namespace blockstore {
+namespace utils {
+
+unique_ref<Block> copyToNewBlock(BlockStore *blockStore, const Block &block) {
+  Data data(block.size());
+  std::memcpy(data.data(), block.data(), block.size());
+  return blockStore->create(data);
+}
+
+void copyTo(Block *target, const Block &source) {
+  ASSERT(target->size() == source.size(), "Can't copy block data when blocks have different sizes");
+  target->write(source.data(), 0, source.size());
+}
+
+void fillWithZeroes(Block *target) {
+  Data zeroes(target->size());
+  zeroes.FillWithZeroes();
+  target->write(zeroes.data(), 0, target->size());
+}
+
+}
+}

src/blockstore/utils/BlockStoreUtils.h

@@ -0,0 +1,19 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_UTILS_BLOCKSTOREUTILS_H_
+#define MESSMER_BLOCKSTORE_UTILS_BLOCKSTOREUTILS_H_
+
+#include <messmer/cpp-utils/pointer/unique_ref.h>
+
+namespace blockstore {
+class BlockStore;
+class Block;
+namespace utils {
+
+cpputils::unique_ref<Block> copyToNewBlock(BlockStore *blockStore, const Block &block);
+void copyTo(Block *target, const Block &source);
+void fillWithZeroes(Block *target);
+
+}
+}
+
+#endif

src/blockstore/utils/FileDoesntExistException.cpp

@@ -0,0 +1,17 @@
+#include "FileDoesntExistException.h"
+
+namespace bf = boost::filesystem;
+
+using std::runtime_error;
+using std::string;
+
+namespace blockstore {
+
+FileDoesntExistException::FileDoesntExistException(const bf::path &filepath)
+: runtime_error(string("The file ")+filepath.c_str()+" doesn't exist") {
+}
+
+FileDoesntExistException::~FileDoesntExistException() {
+}
+
+}

src/blockstore/utils/FileDoesntExistException.h

@@ -0,0 +1,19 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_UTILS_FILEDOESNTEXISTEXCEPTION_H_
+#define MESSMER_BLOCKSTORE_UTILS_FILEDOESNTEXISTEXCEPTION_H_
+
+#include <boost/filesystem/path.hpp>
+
+#include <stdexcept>
+
+namespace blockstore {
+
+class FileDoesntExistException final: public std::runtime_error {
+public:
+  explicit FileDoesntExistException(const boost::filesystem::path &filepath);
+  ~FileDoesntExistException();
+};
+
+}
+
+#endif

src/blockstore/utils/Key.cpp

@@ -0,0 +1 @@
+#include "Key.h"

src/blockstore/utils/Key.h

@@ -0,0 +1,32 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_UTILS_KEY_H_
+#define MESSMER_BLOCKSTORE_UTILS_KEY_H_
+
+#include <string>
+#include <messmer/cpp-utils/data/FixedSizeData.h>
+
+namespace blockstore {
+
+  // A key here is NOT a key for encryption, but a key as used in key->value mappings ("access handle for a block").
+  //TODO Rename to BlockId/BlobId and make it a class containing a FixedSizeData<> member
+  using Key = cpputils::FixedSizeData<16>;
+}
+
+namespace std {
+  //Allow using blockstore::Key in std::unordered_map / std::unordered_set
+  template <> struct hash<blockstore::Key> {
+    size_t operator()(const blockstore::Key &key) const {
+      //Keys are random, so it is enough to use the first few bytes as a hash
+      return *(size_t*)(key.data());
+    }
+  };
+
+  //Allow using blockstore::Key in std::map / std::set
+  template <> struct less<blockstore::Key> {
+    bool operator()(const blockstore::Key &lhs, const blockstore::Key &rhs) const {
+      return 0 > std::memcmp(lhs.data(), rhs.data(), blockstore::Key::BINARY_LENGTH);
+    }
+  };
+}
+
+#endif

test/blockstore/CMakeLists.txt

@@ -0,0 +1,41 @@
+project (blockstore-test)
+
+set(SOURCES
+    utils/BlockStoreUtilsTest.cpp
+    interface/helpers/BlockStoreWithRandomKeysTest.cpp
+    interface/BlockStoreTest.cpp
+    interface/BlockTest.cpp
+    implementations/testfake/TestFakeBlockStoreTest.cpp
+    implementations/inmemory/InMemoryBlockStoreTest.cpp
+    implementations/parallelaccess/ParallelAccessBlockStoreTest.cpp
+    implementations/compressing/CompressingBlockStoreTest.cpp
+    implementations/compressing/compressors/testutils/CompressorTest.cpp
+    implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp
+    implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp
+    implementations/ondisk/OnDiskBlockStoreTest.cpp
+    implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp
+    implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp
+    implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp
+    implementations/caching/CachingBlockStoreTest.cpp
+    implementations/caching/cache/QueueMapTest_Values.cpp
+    implementations/caching/cache/testutils/MinimalKeyType.cpp
+    implementations/caching/cache/testutils/CopyableMovableValueType.cpp
+    implementations/caching/cache/testutils/MinimalValueType.cpp
+    implementations/caching/cache/testutils/QueueMapTest.cpp
+    implementations/caching/cache/testutils/CacheTest.cpp
+    implementations/caching/cache/QueueMapTest_Size.cpp
+    implementations/caching/cache/CacheTest_MoveConstructor.cpp
+    implementations/caching/cache/CacheTest_PushAndPop.cpp
+    implementations/caching/cache/QueueMapTest_MoveConstructor.cpp
+    implementations/caching/cache/QueueMapTest_MemoryLeak.cpp
+    implementations/caching/cache/CacheTest_RaceCondition.cpp
+    implementations/caching/cache/PeriodicTaskTest.cpp
+    implementations/caching/cache/QueueMapTest_Peek.cpp
+)
+
+add_executable(${PROJECT_NAME} ${SOURCES})
+target_link_libraries(${PROJECT_NAME} googletest blockstore)
+add_test(${PROJECT_NAME} ${PROJECT_NAME})
+
+target_enable_style_warnings(${PROJECT_NAME})
+target_activate_cpp14(${PROJECT_NAME})

test/blockstore/implementations/caching/CachingBlockStoreTest.cpp

@@ -0,0 +1,23 @@
+#include "../../../implementations/caching/CachingBlockStore.h"
+#include "../../../implementations/testfake/FakeBlockStore.h"
+#include "../../testutils/BlockStoreTest.h"
+#include "google/gtest/gtest.h"
+#include <messmer/cpp-utils/pointer/unique_ref_boost_optional_gtest_workaround.h>
+
+
+using blockstore::BlockStore;
+using blockstore::caching::CachingBlockStore;
+using blockstore::testfake::FakeBlockStore;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+
+class CachingBlockStoreTestFixture: public BlockStoreTestFixture {
+public:
+  unique_ref<BlockStore> createBlockStore() override {
+    return make_unique_ref<CachingBlockStore>(make_unique_ref<FakeBlockStore>());
+  }
+};
+
+INSTANTIATE_TYPED_TEST_CASE_P(Caching, BlockStoreTest, CachingBlockStoreTestFixture);
+
+//TODO Add specific tests for the blockstore

test/blockstore/implementations/caching/cache/CacheTest_MoveConstructor.cpp

@@ -0,0 +1,35 @@
+#include <google/gtest/gtest.h>
+#include <messmer/cpp-utils/pointer/unique_ref.h>
+#include "../../../../implementations/caching/cache/Cache.h"
+#include "testutils/MinimalKeyType.h"
+#include "testutils/CopyableMovableValueType.h"
+
+using namespace blockstore::caching;
+
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using ::testing::Test;
+
+//Test that Cache uses a move constructor for Value if possible
+class CacheTest_MoveConstructor: public Test {
+public:
+  CacheTest_MoveConstructor(): cache(make_unique_ref<Cache<MinimalKeyType, CopyableMovableValueType, 100>>()) {
+    CopyableMovableValueType::numCopyConstructorCalled = 0;
+  }
+  unique_ref<Cache<MinimalKeyType, CopyableMovableValueType, 100>> cache;
+};
+
+TEST_F(CacheTest_MoveConstructor, MoveIntoCache) {
+  cache->push(MinimalKeyType::create(0), CopyableMovableValueType(2));
+  CopyableMovableValueType val = cache->pop(MinimalKeyType::create(0)).value();
+  val.value(); //Access it to avoid the compiler optimizing the assignment away
+  EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled);
+}
+
+TEST_F(CacheTest_MoveConstructor, CopyIntoCache) {
+  CopyableMovableValueType value(2);
+  cache->push(MinimalKeyType::create(0), value);
+  CopyableMovableValueType val = cache->pop(MinimalKeyType::create(0)).value();
+  val.value(); //Access it to avoid the compiler optimizing the assignment away
+  EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled);
+}

test/blockstore/implementations/caching/cache/CacheTest_PushAndPop.cpp

@@ -0,0 +1,134 @@
+#include "testutils/CacheTest.h"
+
+#include "../../../../implementations/caching/cache/Cache.h"
+#include "testutils/MinimalKeyType.h"
+#include "testutils/MinimalValueType.h"
+#include <messmer/cpp-utils/pointer/unique_ref_boost_optional_gtest_workaround.h>
+
+using ::testing::Test;
+
+using namespace blockstore::caching;
+
+class CacheTest_PushAndPop: public CacheTest {};
+
+TEST_F(CacheTest_PushAndPop, PopNonExistingEntry_EmptyCache) {
+  EXPECT_EQ(boost::none, pop(10));
+}
+
+TEST_F(CacheTest_PushAndPop, PopNonExistingEntry_NonEmptyCache) {
+  push(9, 10);
+  EXPECT_EQ(boost::none, pop(10));
+}
+
+TEST_F(CacheTest_PushAndPop, PopNonExistingEntry_FullCache) {
+  //Add a lot of even numbered keys
+  for (unsigned int i = 0; i < MAX_ENTRIES; ++i) {
+    push(2*i, 2*i);
+  }
+  //Request an odd numbered key
+  EXPECT_EQ(boost::none, pop(9));
+}
+
+TEST_F(CacheTest_PushAndPop, OneEntry) {
+  push(10, 20);
+  EXPECT_EQ(20, pop(10).value());
+}
+
+TEST_F(CacheTest_PushAndPop, MultipleEntries) {
+  push(10, 20);
+  push(20, 30);
+  push(30, 40);
+  EXPECT_EQ(30, pop(20).value());
+  EXPECT_EQ(20, pop(10).value());
+  EXPECT_EQ(40, pop(30).value());
+}
+
+TEST_F(CacheTest_PushAndPop, FullCache) {
+  for(unsigned int i = 0; i < MAX_ENTRIES; ++i) {
+    push(i, 2*i);
+  }
+  for(unsigned int i = 0; i < MAX_ENTRIES; ++i) {
+    EXPECT_EQ(2*i, pop(i).value());
+  }
+}
+
+TEST_F(CacheTest_PushAndPop, FullCache_PushNonOrdered_PopOrdered) {
+  for(unsigned int i = 1; i < MAX_ENTRIES; i += 2) {
+    push(i, 2*i);
+  }
+  for(unsigned int i = 0; i < MAX_ENTRIES; i += 2) {
+    push(i, 2*i);
+  }
+  for(unsigned int i = 0; i < MAX_ENTRIES; ++i) {
+    EXPECT_EQ(2*i, pop(i).value());
+  }
+}
+
+TEST_F(CacheTest_PushAndPop, FullCache_PushOrdered_PopNonOrdered) {
+  for(unsigned int i = 0; i < MAX_ENTRIES; ++i) {
+    push(i, 2*i);
+  }
+  for(unsigned int i = 1; i < MAX_ENTRIES; i += 2) {
+    EXPECT_EQ(2*i, pop(i).value());
+  }
+  for(unsigned int i = 0; i < MAX_ENTRIES; i += 2) {
+    EXPECT_EQ(2*i, pop(i).value());
+  }
+}
+
+int roundDownToEven(int number) {
+  if (number % 2 == 0) {
+    return number;
+  } else {
+    return number - 1;
+  }
+}
+
+int roundDownToOdd(int number) {
+  if (number % 2 != 0) {
+    return number;
+  } else {
+    return number - 1;
+  }
+}
+
+TEST_F(CacheTest_PushAndPop, FullCache_PushNonOrdered_PopNonOrdered) {
+  for(int i = roundDownToEven(MAX_ENTRIES - 1); i >= 0; i -= 2) {
+    push(i, 2*i);
+  }
+  for(unsigned int i = 1; i < MAX_ENTRIES; i += 2) {
+    push(i, 2*i);
+  }
+  for(int i = roundDownToOdd(MAX_ENTRIES-1); i >= 0; i -= 2) {
+    EXPECT_EQ(2*i, pop(i).value());
+  }
+  for(unsigned int i = 0; i < MAX_ENTRIES; i += 2) {
+    EXPECT_EQ(2*i, pop(i).value());
+  }
+}
+
+TEST_F(CacheTest_PushAndPop, MoreThanFullCache) {
+  for(unsigned int i = 0; i < MAX_ENTRIES + 2; ++i) {
+    push(i, 2*i);
+  }
+  //Check that the oldest two elements got deleted automatically
+  EXPECT_EQ(boost::none, pop(0));
+  EXPECT_EQ(boost::none, pop(1));
+  //Check the other elements are still there
+  for(unsigned int i = 2; i < MAX_ENTRIES + 2; ++i) {
+    EXPECT_EQ(2*i, pop(i).value());
+  }
+}
+
+TEST_F(CacheTest_PushAndPop, AfterTimeout) {
+  constexpr double TIMEOUT1_SEC = Cache::MAX_LIFETIME_SEC * 3/4;
+  constexpr double TIMEOUT2_SEC = Cache::PURGE_LIFETIME_SEC * 3/4;
+  static_assert(TIMEOUT1_SEC + TIMEOUT2_SEC > Cache::MAX_LIFETIME_SEC, "Ensure that our chosen timeouts push the first entry out of the cache");
+
+  push(10, 20);
+  boost::this_thread::sleep_for(boost::chrono::milliseconds(static_cast<int>(1000 * TIMEOUT1_SEC)));
+  push(20, 30);
+  boost::this_thread::sleep_for(boost::chrono::milliseconds(static_cast<int>(1000 * TIMEOUT2_SEC)));
+  EXPECT_EQ(boost::none, pop(10));
+  EXPECT_EQ(30, pop(20).value());
+}

test/blockstore/implementations/caching/cache/CacheTest_RaceCondition.cpp

@@ -0,0 +1,109 @@
+#include "testutils/CacheTest.h"
+#include <chrono>
+#include <thread>
+#include <memory>
+#include <future>
+#include <messmer/cpp-utils/lock/ConditionBarrier.h>
+
+using namespace blockstore::caching;
+using std::chrono::seconds;
+using std::string;
+using cpputils::ConditionBarrier;
+using std::unique_ptr;
+using std::make_unique;
+using std::future;
+
+// Regression tests for a race condition.
+// An element could be in the process of being thrown out of the cache and while the destructor is running, another
+// thread calls pop() for the element and gets none returned. But since the destructor isn't finished yet, the data from
+// the cache element also isn't completely written back yet and an application loading it runs into a race condition.
+
+class ObjectWithLongDestructor {
+public:
+    ObjectWithLongDestructor(ConditionBarrier *onDestructorStarted, bool *destructorFinished)
+            : _onDestructorStarted(onDestructorStarted), _destructorFinished(destructorFinished) {}
+    ~ObjectWithLongDestructor() {
+        _onDestructorStarted->release();
+        std::this_thread::sleep_for(seconds(1));
+        *_destructorFinished = true;
+    }
+private:
+    ConditionBarrier *_onDestructorStarted;
+    bool *_destructorFinished;
+
+    DISALLOW_COPY_AND_ASSIGN(ObjectWithLongDestructor);
+};
+
+class CacheTest_RaceCondition: public ::testing::Test {
+public:
+    CacheTest_RaceCondition(): cache(), destructorStarted(), destructorFinished(false) {}
+
+    static constexpr unsigned int MAX_ENTRIES = 100;
+
+    Cache<int, unique_ptr<ObjectWithLongDestructor>, MAX_ENTRIES> cache;
+    ConditionBarrier destructorStarted;
+    bool destructorFinished;
+
+    int pushObjectWithLongDestructor() {
+        cache.push(2, make_unique<ObjectWithLongDestructor>(&destructorStarted, &destructorFinished));
+        return 2;
+    }
+
+    int pushDummyObject() {
+        cache.push(3, nullptr);
+        return 3;
+    }
+
+    future<void> causeCacheOverflowInOtherThread() {
+        //Add maximum+1 element in another thread (this causes the cache to flush the first element in another thread)
+        return std::async(std::launch::async, [this] {
+            for(unsigned int i = 0; i < MAX_ENTRIES+1; ++i) {
+                cache.push(MAX_ENTRIES+i, nullptr);
+            }
+        });
+    }
+
+    void EXPECT_POP_BLOCKS_UNTIL_DESTRUCTOR_FINISHED(int key) {
+        EXPECT_FALSE(destructorFinished);
+        cache.pop(key);
+        EXPECT_TRUE(destructorFinished);
+    }
+
+    void EXPECT_POP_DOESNT_BLOCK_UNTIL_DESTRUCTOR_FINISHED(int key) {
+        EXPECT_FALSE(destructorFinished);
+        cache.pop(key);
+        EXPECT_FALSE(destructorFinished);
+    }
+};
+
+TEST_F(CacheTest_RaceCondition, PopBlocksWhileRequestedElementIsThrownOut_ByAge) {
+    auto id = pushObjectWithLongDestructor();
+
+    destructorStarted.wait();
+    EXPECT_POP_BLOCKS_UNTIL_DESTRUCTOR_FINISHED(id);
+}
+
+TEST_F(CacheTest_RaceCondition, PopDoesntBlockWhileOtherElementIsThrownOut_ByAge) {
+    pushObjectWithLongDestructor();
+    auto id = pushDummyObject();
+
+    destructorStarted.wait();
+    EXPECT_POP_DOESNT_BLOCK_UNTIL_DESTRUCTOR_FINISHED(id);
+}
+
+TEST_F(CacheTest_RaceCondition, PopBlocksWhileRequestedElementIsThrownOut_ByPush) {
+    auto id = pushObjectWithLongDestructor();
+
+    auto future = causeCacheOverflowInOtherThread();
+    destructorStarted.wait();
+    EXPECT_POP_BLOCKS_UNTIL_DESTRUCTOR_FINISHED(id);
+}
+
+TEST_F(CacheTest_RaceCondition, PopDoesntBlockWhileOtherElementIsThrownOut_ByPush) {
+    pushObjectWithLongDestructor();
+    auto id = pushDummyObject();
+
+    auto future = causeCacheOverflowInOtherThread();
+    destructorStarted.wait();
+    EXPECT_POP_DOESNT_BLOCK_UNTIL_DESTRUCTOR_FINISHED(id);
+}

test/blockstore/implementations/caching/cache/PeriodicTaskTest.cpp

@@ -0,0 +1,63 @@
+#include <google/gtest/gtest.h>
+
+#include "../../../../implementations/caching/cache/PeriodicTask.h"
+
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+
+using ::testing::Test;
+using std::mutex;
+using std::unique_lock;
+using std::condition_variable;
+
+using namespace blockstore::caching;
+
+class AtomicCounter {
+public:
+  AtomicCounter(int count): _mutex(), _cv(), _counter(count) {}
+
+  void decrease() {
+    unique_lock<mutex> lock(_mutex);
+    --_counter;
+    _cv.notify_all();
+  }
+
+  void waitForZero() {
+    unique_lock<mutex> lock(_mutex);
+    _cv.wait(lock, [this] () {return _counter <= 0;});
+  }
+private:
+  mutex _mutex;
+  condition_variable _cv;
+  int _counter;
+};
+
+class PeriodicTaskTest: public Test {
+};
+
+TEST_F(PeriodicTaskTest, DoesntDeadlockInDestructorWhenDestructedImmediately) {
+  PeriodicTask task([](){}, 1);
+}
+
+TEST_F(PeriodicTaskTest, CallsCallbackAtLeast10Times) {
+  AtomicCounter counter(10);
+
+  PeriodicTask task([&counter](){
+    counter.decrease();
+  }, 0.001);
+
+  counter.waitForZero();
+}
+
+TEST_F(PeriodicTaskTest, DoesntCallCallbackAfterDestruction) {
+  std::atomic<int> callCount(0);
+  {
+    PeriodicTask task([&callCount](){
+      callCount += 1;
+    }, 0.001);
+  }
+  int callCountDirectlyAfterDestruction = callCount;
+  boost::this_thread::sleep_for(boost::chrono::seconds(1));
+  EXPECT_EQ(callCountDirectlyAfterDestruction, callCount);
+}

test/blockstore/implementations/caching/cache/QueueMapTest_MemoryLeak.cpp

@@ -0,0 +1,87 @@
+#include "testutils/QueueMapTest.h"
+
+// Tests that QueueMap calls destructors correctly.
+// This is needed, because QueueMap does its own memory management.
+class QueueMapTest_MemoryLeak: public QueueMapTest {
+public:
+  void EXPECT_NUM_INSTANCES(int num) {
+    EXPECT_EQ(num, MinimalKeyType::instances);
+    EXPECT_EQ(num, MinimalValueType::instances);
+  }
+};
+
+TEST_F(QueueMapTest_MemoryLeak, Empty) {
+  EXPECT_NUM_INSTANCES(0);
+}
+
+TEST_F(QueueMapTest_MemoryLeak, AfterPushingOne) {
+  push(2, 3);
+  EXPECT_NUM_INSTANCES(1);
+}
+
+TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwo) {
+  push(2, 3);
+  push(3, 4);
+  EXPECT_NUM_INSTANCES(2);
+}
+
+TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwoAndPoppingOldest) {
+  push(2, 3);
+  push(3, 4);
+  pop();
+  EXPECT_NUM_INSTANCES(1);
+}
+
+TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwoAndPoppingFirst) {
+  push(2, 3);
+  push(3, 4);
+  pop(2);
+  EXPECT_NUM_INSTANCES(1);
+}
+
+TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwoAndPoppingLast) {
+  push(2, 3);
+  push(3, 4);
+  pop(3);
+  EXPECT_NUM_INSTANCES(1);
+}
+
+TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOne) {
+  push(2, 3);
+  pop();
+  EXPECT_NUM_INSTANCES(0);
+}
+
+TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePerKey) {
+  push(2, 3);
+  pop(2);
+  EXPECT_NUM_INSTANCES(0);
+}
+
+TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePushingOne) {
+  push(2, 3);
+  pop();
+  push(3, 4);
+  EXPECT_NUM_INSTANCES(1);
+}
+
+TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePerKeyPushingOne) {
+  push(2, 3);
+  pop(2);
+  push(3, 4);
+  EXPECT_NUM_INSTANCES(1);
+}
+
+TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePushingSame) {
+  push(2, 3);
+  pop();
+  push(2, 3);
+  EXPECT_NUM_INSTANCES(1);
+}
+
+TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePerKeyPushingSame) {
+  push(2, 3);
+  pop(2);
+  push(2, 3);
+  EXPECT_NUM_INSTANCES(1);
+}

test/blockstore/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp

@@ -0,0 +1,50 @@
+#include <google/gtest/gtest.h>
+#include <messmer/cpp-utils/pointer/unique_ref.h>
+#include "../../../../implementations/caching/cache/QueueMap.h"
+#include "testutils/MinimalKeyType.h"
+#include "testutils/CopyableMovableValueType.h"
+
+using namespace blockstore::caching;
+
+using ::testing::Test;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+
+//Test that QueueMap uses a move constructor for Value if possible
+class QueueMapTest_MoveConstructor: public Test {
+public:
+  QueueMapTest_MoveConstructor(): map(make_unique_ref<QueueMap<MinimalKeyType, CopyableMovableValueType>>()) {
+    CopyableMovableValueType::numCopyConstructorCalled = 0;
+  }
+  unique_ref<QueueMap<MinimalKeyType, CopyableMovableValueType>> map;
+};
+
+TEST_F(QueueMapTest_MoveConstructor, PushingAndPopping_MoveIntoMap) {
+  map->push(MinimalKeyType::create(0), CopyableMovableValueType(2));
+  CopyableMovableValueType val = map->pop().value();
+  val.value(); //Access it to avoid the compiler optimizing the assignment away
+  EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled);
+}
+
+TEST_F(QueueMapTest_MoveConstructor, PushingAndPoppingPerKey_MoveIntoMap) {
+  map->push(MinimalKeyType::create(0), CopyableMovableValueType(2));
+  CopyableMovableValueType val = map->pop(MinimalKeyType::create(0)).value();
+  val.value(); //Access it to avoid the compiler optimizing the assignment away
+  EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled);
+}
+
+TEST_F(QueueMapTest_MoveConstructor, PushingAndPopping_CopyIntoMap) {
+  CopyableMovableValueType value(2);
+  map->push(MinimalKeyType::create(0), value);
+  CopyableMovableValueType val = map->pop().value();
+  val.value(); //Access it to avoid the compiler optimizing the assignment away
+  EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled);
+}
+
+TEST_F(QueueMapTest_MoveConstructor, PushingAndPoppingPerKey_CopyIntoMap) {
+  CopyableMovableValueType value(2);
+  map->push(MinimalKeyType::create(0), value);
+  CopyableMovableValueType val = map->pop(MinimalKeyType::create(0)).value();
+  val.value(); //Access it to avoid the compiler optimizing the assignment away
+  EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled);
+}

test/blockstore/implementations/caching/cache/QueueMapTest_Peek.cpp

@@ -0,0 +1,34 @@
+#include "testutils/QueueMapTest.h"
+#include <boost/optional/optional_io.hpp>
+
+class QueueMapPeekTest: public QueueMapTest {};
+
+TEST_F(QueueMapPeekTest, PoppingFromEmpty) {
+  EXPECT_EQ(boost::none, peek());
+}
+
+TEST_F(QueueMapPeekTest, PushingOne) {
+  push(3, 2);
+  EXPECT_EQ(2, peek().value());
+}
+
+TEST_F(QueueMapPeekTest, PushingTwo) {
+  push(2, 3);
+  push(3, 4);
+  EXPECT_EQ(3, peek().value());
+  EXPECT_EQ(3, peek().value());
+  EXPECT_EQ(3, pop().value());
+  EXPECT_EQ(4, peek().value());
+  EXPECT_EQ(4, peek().value());
+  EXPECT_EQ(4, pop().value());
+  EXPECT_EQ(boost::none, peek());
+  EXPECT_EQ(boost::none, pop());
+}
+
+TEST_F(QueueMapPeekTest, AfterPushingTwoAndPoppingFirst) {
+  push(2, 3);
+  push(3, 4);
+  pop(2);
+  EXPECT_EQ(boost::none, pop(2));
+  EXPECT_EQ(4, peek().value());
+}

test/blockstore/implementations/caching/cache/QueueMapTest_Size.cpp

@@ -0,0 +1,79 @@
+#include "testutils/QueueMapTest.h"
+
+class QueueMapTest_Size: public QueueMapTest {};
+
+TEST_F(QueueMapTest_Size, Empty) {
+  EXPECT_EQ(0, size());
+}
+
+TEST_F(QueueMapTest_Size, AfterPushingOne) {
+  push(2, 3);
+  EXPECT_EQ(1, size());
+}
+
+TEST_F(QueueMapTest_Size, AfterPushingTwo) {
+  push(2, 3);
+  push(3, 4);
+  EXPECT_EQ(2, size());
+}
+
+TEST_F(QueueMapTest_Size, AfterPushingTwoAndPoppingOldest) {
+  push(2, 3);
+  push(3, 4);
+  pop();
+  EXPECT_EQ(1, size());
+}
+
+TEST_F(QueueMapTest_Size, AfterPushingTwoAndPoppingFirst) {
+  push(2, 3);
+  push(3, 4);
+  pop(2);
+  EXPECT_EQ(1, size());
+}
+
+TEST_F(QueueMapTest_Size, AfterPushingTwoAndPoppingLast) {
+  push(2, 3);
+  push(3, 4);
+  pop(3);
+  EXPECT_EQ(1, size());
+}
+
+TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOne) {
+  push(2, 3);
+  pop();
+  EXPECT_EQ(0, size());
+}
+
+TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePerKey) {
+  push(2, 3);
+  pop(2);
+  EXPECT_EQ(0, size());
+}
+
+TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePushingOne) {
+  push(2, 3);
+  pop();
+  push(3, 4);
+  EXPECT_EQ(1, size());
+}
+
+TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePerKeyPushingOne) {
+  push(2, 3);
+  pop(2);
+  push(3, 4);
+  EXPECT_EQ(1, size());
+}
+
+TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePushingSame) {
+  push(2, 3);
+  pop();
+  push(2, 3);
+  EXPECT_EQ(1, size());
+}
+
+TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePerKeyPushingSame) {
+  push(2, 3);
+  pop(2);
+  push(2, 3);
+  EXPECT_EQ(1, size());
+}

test/blockstore/implementations/caching/cache/QueueMapTest_Values.cpp

@@ -0,0 +1,151 @@
+#include "testutils/QueueMapTest.h"
+#include <messmer/cpp-utils/pointer/unique_ref_boost_optional_gtest_workaround.h>
+
+class QueueMapTest_Values: public QueueMapTest {};
+
+TEST_F(QueueMapTest_Values, PoppingFromEmpty) {
+  EXPECT_EQ(boost::none, pop());
+}
+
+TEST_F(QueueMapTest_Values, PoppingFromEmptyPerKey) {
+  EXPECT_EQ(boost::none, pop(2));
+}
+
+TEST_F(QueueMapTest_Values, PoppingNonexistingPerKey) {
+  push(3, 2);
+  EXPECT_EQ(boost::none, pop(2));
+}
+
+TEST_F(QueueMapTest_Values, PushingOne) {
+  push(3, 2);
+  EXPECT_EQ(2, pop(3).value());
+  EXPECT_EQ(boost::none, pop());
+}
+
+TEST_F(QueueMapTest_Values, PushingTwo) {
+  push(2, 3);
+  push(3, 4);
+  EXPECT_EQ(3, pop().value());
+  EXPECT_EQ(4, pop().value());
+  EXPECT_EQ(boost::none, pop());
+}
+
+TEST_F(QueueMapTest_Values, AfterPushingTwoAndPoppingFirst) {
+  push(2, 3);
+  push(3, 4);
+  pop(2);
+  EXPECT_EQ(boost::none, pop(2));
+  EXPECT_EQ(4, pop(3).value());
+  EXPECT_EQ(boost::none, pop());
+}
+
+TEST_F(QueueMapTest_Values, AfterPushingTwoAndPoppingLast) {
+  push(2, 3);
+  push(3, 4);
+  pop(3);
+  EXPECT_EQ(boost::none, pop(3));
+  EXPECT_EQ(3, pop(2).value());
+  EXPECT_EQ(boost::none, pop());
+}
+
+TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOne) {
+  push(2, 3);
+  pop();
+  EXPECT_EQ(boost::none, pop());
+  EXPECT_EQ(boost::none, pop(2));
+}
+
+TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOnePerKey) {
+  push(2, 3);
+  pop(2);
+  EXPECT_EQ(boost::none, pop());
+  EXPECT_EQ(boost::none, pop(2));
+}
+
+TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOnePushingOne) {
+  push(2, 3);
+  pop();
+  push(3, 4);
+  EXPECT_EQ(boost::none, pop(2));
+  EXPECT_EQ(4, pop(3).value());
+  EXPECT_EQ(boost::none, pop());
+}
+
+TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOnePerKeyPushingOne) {
+  push(2, 3);
+  pop(2);
+  push(3, 4);
+  EXPECT_EQ(boost::none, pop(2));
+  EXPECT_EQ(4, pop(3).value());
+  EXPECT_EQ(boost::none, pop());
+}
+
+TEST_F(QueueMapTest_Values, PushingSomePoppingMiddlePerKey) {
+  push(1, 2);
+  push(2, 3);
+  push(3, 4);
+  push(4, 5);
+  push(5, 6);
+  EXPECT_EQ(3, pop(2).value());
+  EXPECT_EQ(5, pop(4).value());
+  EXPECT_EQ(2, pop().value());
+  EXPECT_EQ(4, pop().value());
+  EXPECT_EQ(6, pop().value());
+  EXPECT_EQ(boost::none, pop());
+}
+
+TEST_F(QueueMapTest_Values, PushingSomePoppingFirstPerKey) {
+  push(1, 2);
+  push(2, 3);
+  push(3, 4);
+  push(4, 5);
+  push(5, 6);
+  EXPECT_EQ(2, pop(1).value());
+  EXPECT_EQ(3, pop(2).value());
+  EXPECT_EQ(4, pop().value());
+  EXPECT_EQ(5, pop().value());
+  EXPECT_EQ(6, pop().value());
+  EXPECT_EQ(boost::none, pop());
+}
+
+TEST_F(QueueMapTest_Values, PushingSomePoppingLastPerKey) {
+  push(1, 2);
+  push(2, 3);
+  push(3, 4);
+  push(4, 5);
+  push(5, 6);
+  EXPECT_EQ(6, pop(5).value());
+  EXPECT_EQ(5, pop(4).value());
+  EXPECT_EQ(2, pop().value());
+  EXPECT_EQ(3, pop().value());
+  EXPECT_EQ(4, pop().value());
+  EXPECT_EQ(boost::none, pop());
+}
+
+//This test forces the underlying datastructure (std::map or std::unordered_map) to grow and reallocate memory.
+//So it tests, that QueueMap still works after reallocating memory.
+TEST_F(QueueMapTest_Values, ManyValues) {
+  //Push 1 million entries
+  for (int i = 0; i < 1000000; ++i) {
+    push(i, 2*i);
+  }
+  //pop every other one by key
+  for (int i = 0; i < 1000000; i += 2) {
+    EXPECT_EQ(2*i, pop(i).value());
+  }
+  //pop the rest in queue order
+  for (int i = 1; i < 1000000; i += 2) {
+    EXPECT_EQ(2*i, peek().value());
+    EXPECT_EQ(2*i, pop().value());
+  }
+  EXPECT_EQ(0, size());
+  EXPECT_EQ(boost::none, pop());
+  EXPECT_EQ(boost::none, peek());
+}
+
+TEST_F(QueueMapTest_Values, PushAlreadyExistingValue) {
+  push(2, 3);
+  EXPECT_ANY_THROW(
+    push(2, 4);
+  );
+}

test/blockstore/implementations/caching/cache/testutils/CacheTest.cpp

@@ -0,0 +1,13 @@
+#include "CacheTest.h"
+
+void CacheTest::push(int key, int value) {
+  return _cache.push(MinimalKeyType::create(key), MinimalValueType::create(value));
+}
+
+boost::optional<int> CacheTest::pop(int key) {
+  boost::optional<MinimalValueType> entry = _cache.pop(MinimalKeyType::create(key));
+  if (!entry) {
+    return boost::none;
+  }
+  return entry->value();
+}

test/blockstore/implementations/caching/cache/testutils/CacheTest.h

@@ -0,0 +1,29 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_
+#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_
+
+#include <google/gtest/gtest.h>
+#include "../../../../../implementations/caching/cache/Cache.h"
+#include "MinimalKeyType.h"
+#include "MinimalValueType.h"
+#include <boost/optional.hpp>
+
+// This class is a parent class for tests on QueueMap.
+// It offers functions to work with a QueueMap test object which is built using types having only the minimal type requirements.
+// Furthermore, the class checks that there are no memory leaks left after destructing the QueueMap (by counting leftover instances of Keys/Values).
+class CacheTest: public ::testing::Test {
+public:
+  CacheTest(): _cache() {}
+
+  void push(int key, int value);
+  boost::optional<int> pop(int key);
+
+  static constexpr unsigned int MAX_ENTRIES = 100;
+
+  using Cache = blockstore::caching::Cache<MinimalKeyType, MinimalValueType, MAX_ENTRIES>;
+
+private:
+  Cache _cache;
+};
+
+#endif

test/blockstore/implementations/caching/cache/testutils/CopyableMovableValueType.cpp

@@ -0,0 +1,3 @@
+#include "CopyableMovableValueType.h"
+
+int CopyableMovableValueType::numCopyConstructorCalled = 0;

test/blockstore/implementations/caching/cache/testutils/CopyableMovableValueType.h

@@ -0,0 +1,33 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_COPYABLEMOVABLEVALUETYPE_H_
+#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_COPYABLEMOVABLEVALUETYPE_H_
+
+class CopyableMovableValueType {
+public:
+  static int numCopyConstructorCalled;
+  CopyableMovableValueType(int value): _value(value) {}
+  CopyableMovableValueType(const CopyableMovableValueType &rhs): CopyableMovableValueType(rhs._value) {
+    ++numCopyConstructorCalled;
+  }
+  CopyableMovableValueType &operator=(const CopyableMovableValueType &rhs) {
+    _value = rhs._value;
+    ++numCopyConstructorCalled;
+    return *this;
+  }
+  CopyableMovableValueType(CopyableMovableValueType &&rhs): CopyableMovableValueType(rhs._value) {
+    //Don't increase numCopyConstructorCalled
+  }
+  CopyableMovableValueType &operator=(CopyableMovableValueType &&rhs) {
+    //Don't increase numCopyConstructorCalled
+    _value = rhs._value;
+    return *this;
+  }
+  int value() const {
+    return _value;
+  }
+private:
+  int _value;
+};
+
+
+#endif

test/blockstore/implementations/caching/cache/testutils/MinimalKeyType.cpp

@@ -0,0 +1,3 @@
+#include "MinimalKeyType.h"
+
+int MinimalKeyType::instances = 0;

test/blockstore/implementations/caching/cache/testutils/MinimalKeyType.h

@@ -0,0 +1,47 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALKEYTYPE_H_
+#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALKEYTYPE_H_
+
+#include <unordered_map>
+
+// This is a not-default-constructible Key type
+class MinimalKeyType {
+public:
+  static int instances;
+
+  static MinimalKeyType create(int value) {
+    return MinimalKeyType(value);
+  }
+
+  MinimalKeyType(const MinimalKeyType &rhs): MinimalKeyType(rhs.value()) {
+  }
+
+  ~MinimalKeyType() {
+    --instances;
+  }
+
+  int value() const {
+    return _value;
+  }
+
+private:
+  MinimalKeyType(int value): _value(value) {
+    ++instances;
+  }
+
+  int _value;
+};
+
+namespace std {
+template <> struct hash<MinimalKeyType> {
+  size_t operator()(const MinimalKeyType &obj) const {
+    return obj.value();
+  }
+};
+}
+
+inline bool operator==(const MinimalKeyType &lhs, const MinimalKeyType &rhs) {
+  return lhs.value() == rhs.value();
+}
+
+#endif

test/blockstore/implementations/caching/cache/testutils/MinimalValueType.cpp

@@ -0,0 +1,3 @@
+#include "MinimalValueType.h"
+
+int MinimalValueType::instances = 0;

test/blockstore/implementations/caching/cache/testutils/MinimalValueType.h

@@ -0,0 +1,52 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALVALUETYPE_H_
+#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALVALUETYPE_H_
+
+#include <messmer/cpp-utils/macros.h>
+#include <cassert>
+#include <messmer/cpp-utils/assert/assert.h>
+
+// This is a not-default-constructible non-copyable but moveable Value type
+class MinimalValueType {
+public:
+  static int instances;
+
+  static MinimalValueType create(int value) {
+    return MinimalValueType(value);
+  }
+
+  MinimalValueType(MinimalValueType &&rhs): MinimalValueType(rhs.value()) {
+    rhs._isMoved = true;
+  }
+
+  MinimalValueType &operator=(MinimalValueType &&rhs) {
+    _value = rhs.value();
+    _isMoved = false;
+    rhs._isMoved = true;
+    return *this;
+  }
+
+  ~MinimalValueType() {
+    ASSERT(!_isDestructed, "Object was already destructed before");
+    --instances;
+    _isDestructed = true;
+  }
+
+  int value() const {
+    ASSERT(!_isMoved && !_isDestructed, "Object is invalid");
+    return _value;
+  }
+
+private:
+  MinimalValueType(int value): _value(value), _isMoved(false), _isDestructed(false) {
+    ++instances;
+  }
+
+  int _value;
+  bool _isMoved;
+  bool _isDestructed;
+
+  DISALLOW_COPY_AND_ASSIGN(MinimalValueType);
+};
+
+#endif

test/blockstore/implementations/caching/cache/testutils/QueueMapTest.cpp

@@ -0,0 +1,44 @@
+#include "QueueMapTest.h"
+
+QueueMapTest::QueueMapTest(): _map(cpputils::make_unique_ref<blockstore::caching::QueueMap<MinimalKeyType, MinimalValueType>>()) {
+  MinimalKeyType::instances = 0;
+  MinimalValueType::instances = 0;
+}
+
+QueueMapTest::~QueueMapTest() {
+  cpputils::destruct(std::move(_map));
+  EXPECT_EQ(0, MinimalKeyType::instances);
+  EXPECT_EQ(0, MinimalValueType::instances);
+}
+
+void QueueMapTest::push(int key, int value) {
+  _map->push(MinimalKeyType::create(key), MinimalValueType::create(value));
+}
+
+boost::optional<int> QueueMapTest::pop() {
+  auto elem = _map->pop();
+  if (!elem) {
+    return boost::none;
+  }
+  return elem.value().value();
+}
+
+boost::optional<int> QueueMapTest::pop(int key) {
+  auto elem = _map->pop(MinimalKeyType::create(key));
+  if (!elem) {
+    return boost::none;
+  }
+  return elem.value().value();
+}
+
+boost::optional<int> QueueMapTest::peek() {
+  auto elem = _map->peek();
+  if (!elem) {
+    return boost::none;
+  }
+  return elem.value().value();
+}
+
+int QueueMapTest::size() {
+  return _map->size();
+}

test/blockstore/implementations/caching/cache/testutils/QueueMapTest.h

@@ -0,0 +1,31 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_
+#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_
+
+#include <google/gtest/gtest.h>
+#include <messmer/cpp-utils/pointer/unique_ref.h>
+#include "../../../../../implementations/caching/cache/QueueMap.h"
+#include "MinimalKeyType.h"
+#include "MinimalValueType.h"
+#include <boost/optional.hpp>
+
+// This class is a parent class for tests on QueueMap.
+// It offers functions to work with a QueueMap test object which is built using types having only the minimal type requirements.
+// Furthermore, the class checks that there are no memory leaks left after destructing the QueueMap (by counting leftover instances of Keys/Values).
+class QueueMapTest: public ::testing::Test {
+public:
+  QueueMapTest();
+  ~QueueMapTest();
+
+  void push(int key, int value);
+  boost::optional<int> pop();
+  boost::optional<int> pop(int key);
+  boost::optional<int> peek();
+  int size();
+
+private:
+  cpputils::unique_ref<blockstore::caching::QueueMap<MinimalKeyType, MinimalValueType>> _map;
+};
+
+
+#endif

test/blockstore/implementations/compressing/CompressingBlockStoreTest.cpp

@@ -0,0 +1,28 @@
+#include "../../../implementations/compressing/CompressingBlockStore.h"
+#include "../../../implementations/compressing/compressors/Gzip.h"
+#include "../../../implementations/compressing/compressors/RunLengthEncoding.h"
+#include "../../../implementations/testfake/FakeBlockStore.h"
+#include "../../testutils/BlockStoreTest.h"
+#include "google/gtest/gtest.h"
+
+using ::testing::Test;
+
+using blockstore::BlockStore;
+using blockstore::compressing::CompressingBlockStore;
+using blockstore::compressing::Gzip;
+using blockstore::compressing::RunLengthEncoding;
+using blockstore::testfake::FakeBlockStore;
+
+using cpputils::make_unique_ref;
+using cpputils::unique_ref;
+
+template<class Compressor>
+class CompressingBlockStoreTestFixture: public BlockStoreTestFixture {
+public:
+  unique_ref<BlockStore> createBlockStore() override {
+    return make_unique_ref<CompressingBlockStore<Compressor>>(make_unique_ref<FakeBlockStore>());
+  }
+};
+
+INSTANTIATE_TYPED_TEST_CASE_P(Compressing_Gzip, BlockStoreTest, CompressingBlockStoreTestFixture<Gzip>);
+INSTANTIATE_TYPED_TEST_CASE_P(Compressing_RunLengthEncoding, BlockStoreTest, CompressingBlockStoreTestFixture<RunLengthEncoding>);

test/blockstore/implementations/compressing/compressors/testutils/CompressorTest.cpp

@@ -0,0 +1,92 @@
+#include <google/gtest/gtest.h>
+#include "../../../../../implementations/compressing/compressors/Gzip.h"
+#include "../../../../../implementations/compressing/compressors/RunLengthEncoding.h"
+#include <messmer/cpp-utils/data/DataFixture.h>
+
+using namespace blockstore::compressing;
+using cpputils::Data;
+using cpputils::DataFixture;
+
+template<class Compressor>
+class CompressorTest: public ::testing::Test {
+public:
+    void EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(const Data &data) {
+        Data compressed = Compressor::Compress(data);
+        Data decompressed = Compressor::Decompress(compressed.data(), compressed.size());
+        EXPECT_EQ(data, decompressed);
+    }
+};
+
+TYPED_TEST_CASE_P(CompressorTest);
+
+TYPED_TEST_P(CompressorTest, Empty) {
+  Data empty(0);
+  this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(empty);
+}
+
+TYPED_TEST_P(CompressorTest, ArbitraryData) {
+  Data fixture = DataFixture::generate(10240);
+  this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(fixture);
+}
+
+TYPED_TEST_P(CompressorTest, Zeroes) {
+  Data zeroes(10240);
+  zeroes.FillWithZeroes();
+  this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(zeroes);
+}
+
+TYPED_TEST_P(CompressorTest, Runs) {
+    Data data(4096);
+    std::memset(data.dataOffset(0),    0xF2, 1024);
+    std::memset(data.dataOffset(1024), 0x00, 1024);
+    std::memset(data.dataOffset(2048), 0x01, 2048);
+    this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(data);
+}
+
+TYPED_TEST_P(CompressorTest, RunsAndArbitrary) {
+    Data data(4096);
+    std::memset(data.dataOffset(0),    0xF2, 1024);
+    std::memcpy(data.dataOffset(1024), DataFixture::generate(1024).data(), 1024);
+    std::memset(data.dataOffset(2048), 0x01, 2048);
+    this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(data);
+}
+
+TYPED_TEST_P(CompressorTest, LargeData) {
+    // this is larger than what fits into 16bit (16bit are for example used as run length indicator in RunLengthEncoding)
+    Data fixture = DataFixture::generate(200000);
+    this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(fixture);
+}
+
+TYPED_TEST_P(CompressorTest, LargeRuns) {
+    // each run is larger than what fits into 16bit (16bit are for example used as run length indicator in RunLengthEncoding)
+    constexpr size_t RUN_SIZE = 200000;
+    Data data(3*RUN_SIZE);
+    std::memset(data.dataOffset(0),          0xF2, RUN_SIZE);
+    std::memset(data.dataOffset(RUN_SIZE),   0x00, RUN_SIZE);
+    std::memset(data.dataOffset(2*RUN_SIZE), 0x01, RUN_SIZE);
+    this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(data);
+}
+
+TYPED_TEST_P(CompressorTest, LargeRunsAndArbitrary) {
+    // each run is larger than what fits into 16bit (16bit are for example used as run length indicator in RunLengthEncoding)
+    constexpr size_t RUN_SIZE = 200000;
+    Data data(3*RUN_SIZE);
+    std::memset(data.dataOffset(0), 0xF2, RUN_SIZE);
+    std::memcpy(data.dataOffset(RUN_SIZE), DataFixture::generate(RUN_SIZE).data(), RUN_SIZE);
+    std::memset(data.dataOffset(2*RUN_SIZE), 0x01, RUN_SIZE);
+    this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(data);
+}
+
+REGISTER_TYPED_TEST_CASE_P(CompressorTest,
+        Empty,
+        ArbitraryData,
+        Zeroes,
+        Runs,
+        RunsAndArbitrary,
+        LargeData,
+        LargeRuns,
+        LargeRunsAndArbitrary
+);
+
+INSTANTIATE_TYPED_TEST_CASE_P(Gzip, CompressorTest, Gzip);
+INSTANTIATE_TYPED_TEST_CASE_P(RunLengthEncoding, CompressorTest, RunLengthEncoding);

test/blockstore/implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp

@@ -0,0 +1,39 @@
+#include <messmer/cpp-utils/crypto/symmetric/ciphers.h>
+#include <messmer/cpp-utils/crypto/symmetric/Cipher.h>
+#include "../../../implementations/encrypted/EncryptedBlockStore.h"
+#include "../../../implementations/testfake/FakeBlockStore.h"
+#include "../../testutils/BlockStoreTest.h"
+#include <messmer/cpp-utils/test/crypto/symmetric/testutils/FakeAuthenticatedCipher.h>
+#include "google/gtest/gtest.h"
+
+using ::testing::Test;
+
+using blockstore::BlockStore;
+using blockstore::encrypted::EncryptedBlockStore;
+using blockstore::testfake::FakeBlockStore;
+using cpputils::AES256_GCM;
+using cpputils::AES256_CFB;
+using cpputils::FakeAuthenticatedCipher;
+
+using cpputils::Data;
+using cpputils::DataFixture;
+using cpputils::make_unique_ref;
+using cpputils::unique_ref;
+
+template<class Cipher>
+class EncryptedBlockStoreTestFixture: public BlockStoreTestFixture {
+public:
+  unique_ref<BlockStore> createBlockStore() override {
+    return make_unique_ref<EncryptedBlockStore<Cipher>>(make_unique_ref<FakeBlockStore>(), createKeyFixture());
+  }
+
+private:
+  static typename Cipher::EncryptionKey createKeyFixture(int seed = 0) {
+    Data data = DataFixture::generate(Cipher::EncryptionKey::BINARY_LENGTH, seed);
+    return Cipher::EncryptionKey::FromBinary(data.data());
+  }
+};
+
+INSTANTIATE_TYPED_TEST_CASE_P(Encrypted_FakeCipher, BlockStoreTest, EncryptedBlockStoreTestFixture<FakeAuthenticatedCipher>);
+INSTANTIATE_TYPED_TEST_CASE_P(Encrypted_AES256_GCM, BlockStoreTest, EncryptedBlockStoreTestFixture<AES256_GCM>);
+INSTANTIATE_TYPED_TEST_CASE_P(Encrypted_AES256_CFB, BlockStoreTest, EncryptedBlockStoreTestFixture<AES256_CFB>);

test/blockstore/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp

@@ -0,0 +1,114 @@
+#include <google/gtest/gtest.h>
+#include <messmer/cpp-utils/test/crypto/symmetric/testutils/FakeAuthenticatedCipher.h>
+#include "../../../implementations/encrypted/EncryptedBlockStore.h"
+#include "../../../implementations/testfake/FakeBlockStore.h"
+#include "../../../utils/BlockStoreUtils.h"
+#include <messmer/cpp-utils/data/DataFixture.h>
+
+using ::testing::Test;
+
+using cpputils::DataFixture;
+using cpputils::Data;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using cpputils::FakeAuthenticatedCipher;
+
+using blockstore::testfake::FakeBlockStore;
+
+using namespace blockstore::encrypted;
+
+class EncryptedBlockStoreTest: public Test {
+public:
+  static constexpr unsigned int BLOCKSIZE = 1024;
+  EncryptedBlockStoreTest():
+    baseBlockStore(new FakeBlockStore),
+    blockStore(make_unique_ref<EncryptedBlockStore<FakeAuthenticatedCipher>>(std::move(cpputils::nullcheck(std::unique_ptr<FakeBlockStore>(baseBlockStore)).value()), FakeAuthenticatedCipher::Key1())),
+    data(DataFixture::generate(BLOCKSIZE)) {
+  }
+  FakeBlockStore *baseBlockStore;
+  unique_ref<EncryptedBlockStore<FakeAuthenticatedCipher>> blockStore;
+  Data data;
+
+  blockstore::Key CreateBlockDirectlyWithFixtureAndReturnKey() {
+    return blockStore->create(data)->key();
+  }
+
+  blockstore::Key CreateBlockWriteFixtureToItAndReturnKey() {
+    auto block = blockStore->create(Data(data.size()));
+    block->write(data.data(), 0, data.size());
+    return block->key();
+  }
+
+  void ModifyBaseBlock(const blockstore::Key &key) {
+    auto block = baseBlockStore->load(key).value();
+    uint8_t middle_byte = ((byte*)block->data())[10];
+    uint8_t new_middle_byte = middle_byte + 1;
+    block->write(&new_middle_byte, 10, 1);
+  }
+
+  blockstore::Key CopyBaseBlock(const blockstore::Key &key) {
+    auto source = baseBlockStore->load(key).value();
+    return blockstore::utils::copyToNewBlock(baseBlockStore, *source)->key();
+  }
+
+private:
+  DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStoreTest);
+};
+
+TEST_F(EncryptedBlockStoreTest, LoadingWithSameKeyWorks_WriteOnCreate) {
+  auto key = CreateBlockDirectlyWithFixtureAndReturnKey();
+  auto loaded = blockStore->load(key);
+  EXPECT_NE(boost::none, loaded);
+  EXPECT_EQ(data.size(), (*loaded)->size());
+  EXPECT_EQ(0, std::memcmp(data.data(), (*loaded)->data(), data.size()));
+}
+
+TEST_F(EncryptedBlockStoreTest, LoadingWithSameKeyWorks_WriteSeparately) {
+  auto key = CreateBlockWriteFixtureToItAndReturnKey();
+  auto loaded = blockStore->load(key);
+  EXPECT_NE(boost::none, loaded);
+  EXPECT_EQ(data.size(), (*loaded)->size());
+  EXPECT_EQ(0, std::memcmp(data.data(), (*loaded)->data(), data.size()));
+}
+
+TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentKeyDoesntWork_WriteOnCreate) {
+  auto key = CreateBlockDirectlyWithFixtureAndReturnKey();
+  blockStore->__setKey(FakeAuthenticatedCipher::Key2());
+  auto loaded = blockStore->load(key);
+  EXPECT_EQ(boost::none, loaded);
+}
+
+TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentKeyDoesntWork_WriteSeparately) {
+  auto key = CreateBlockWriteFixtureToItAndReturnKey();
+  blockStore->__setKey(FakeAuthenticatedCipher::Key2());
+  auto loaded = blockStore->load(key);
+  EXPECT_EQ(boost::none, loaded);
+}
+
+TEST_F(EncryptedBlockStoreTest, LoadingModifiedBlockFails_WriteOnCreate) {
+  auto key = CreateBlockDirectlyWithFixtureAndReturnKey();
+  ModifyBaseBlock(key);
+  auto loaded = blockStore->load(key);
+  EXPECT_EQ(boost::none, loaded);
+}
+
+TEST_F(EncryptedBlockStoreTest, LoadingModifiedBlockFails_WriteSeparately) {
+  auto key = CreateBlockWriteFixtureToItAndReturnKey();
+  ModifyBaseBlock(key);
+  auto loaded = blockStore->load(key);
+  EXPECT_EQ(boost::none, loaded);
+}
+
+TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentBlockIdFails_WriteOnCreate) {
+  auto key = CreateBlockDirectlyWithFixtureAndReturnKey();
+  auto key2 = CopyBaseBlock(key);
+  auto loaded = blockStore->load(key2);
+  EXPECT_EQ(boost::none, loaded);
+}
+
+TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentBlockIdFails_WriteSeparately) {
+  auto key = CreateBlockWriteFixtureToItAndReturnKey();
+  auto key2 = CopyBaseBlock(key);
+  auto loaded = blockStore->load(key2);
+  EXPECT_EQ(boost::none, loaded);
+}

test/blockstore/implementations/inmemory/InMemoryBlockStoreTest.cpp

@@ -0,0 +1,31 @@
+#include "../../../implementations/inmemory/InMemoryBlock.h"
+#include "../../../implementations/inmemory/InMemoryBlockStore.h"
+#include "../../testutils/BlockStoreTest.h"
+#include "../../testutils/BlockStoreWithRandomKeysTest.h"
+#include "google/gtest/gtest.h"
+#include <messmer/cpp-utils/pointer/unique_ref_boost_optional_gtest_workaround.h>
+
+
+using blockstore::BlockStore;
+using blockstore::BlockStoreWithRandomKeys;
+using blockstore::inmemory::InMemoryBlockStore;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+
+class InMemoryBlockStoreTestFixture: public BlockStoreTestFixture {
+public:
+  unique_ref<BlockStore> createBlockStore() override {
+    return make_unique_ref<InMemoryBlockStore>();
+  }
+};
+
+INSTANTIATE_TYPED_TEST_CASE_P(InMemory, BlockStoreTest, InMemoryBlockStoreTestFixture);
+
+class InMemoryBlockStoreWithRandomKeysTestFixture: public BlockStoreWithRandomKeysTestFixture {
+public:
+  unique_ref<BlockStoreWithRandomKeys> createBlockStore() override {
+    return make_unique_ref<InMemoryBlockStore>();
+  }
+};
+
+INSTANTIATE_TYPED_TEST_CASE_P(InMemory, BlockStoreWithRandomKeysTest, InMemoryBlockStoreWithRandomKeysTestFixture);

test/blockstore/implementations/ondisk/OnDiskBlockStoreTest.cpp

@@ -0,0 +1,42 @@
+#include "../../../implementations/ondisk/OnDiskBlock.h"
+#include "../../../implementations/ondisk/OnDiskBlockStore.h"
+#include "../../testutils/BlockStoreTest.h"
+#include "../../testutils/BlockStoreWithRandomKeysTest.h"
+#include "google/gtest/gtest.h"
+
+#include "messmer/cpp-utils/tempfile/TempDir.h"
+
+
+using blockstore::BlockStore;
+using blockstore::BlockStoreWithRandomKeys;
+using blockstore::ondisk::OnDiskBlockStore;
+
+using cpputils::TempDir;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+
+class OnDiskBlockStoreTestFixture: public BlockStoreTestFixture {
+public:
+  OnDiskBlockStoreTestFixture(): tempdir() {}
+
+  unique_ref<BlockStore> createBlockStore() override {
+    return make_unique_ref<OnDiskBlockStore>(tempdir.path());
+  }
+private:
+  TempDir tempdir;
+};
+
+INSTANTIATE_TYPED_TEST_CASE_P(OnDisk, BlockStoreTest, OnDiskBlockStoreTestFixture);
+
+class OnDiskBlockStoreWithRandomKeysTestFixture: public BlockStoreWithRandomKeysTestFixture {
+public:
+  OnDiskBlockStoreWithRandomKeysTestFixture(): tempdir() {}
+  
+  unique_ref<BlockStoreWithRandomKeys> createBlockStore() override {
+    return make_unique_ref<OnDiskBlockStore>(tempdir.path());
+  }
+private:
+  TempDir tempdir;
+};
+
+INSTANTIATE_TYPED_TEST_CASE_P(OnDisk, BlockStoreWithRandomKeysTest, OnDiskBlockStoreWithRandomKeysTestFixture);

test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp

@@ -0,0 +1,83 @@
+#include "../../../../implementations/ondisk/OnDiskBlock.h"
+#include "google/gtest/gtest.h"
+
+#include <messmer/cpp-utils/tempfile/TempFile.h>
+#include <messmer/cpp-utils/tempfile/TempDir.h>
+
+using ::testing::Test;
+using ::testing::WithParamInterface;
+using ::testing::Values;
+using cpputils::Data;
+using cpputils::TempFile;
+using cpputils::TempDir;
+using cpputils::unique_ref;
+
+using namespace blockstore;
+using namespace blockstore::ondisk;
+
+namespace bf = boost::filesystem;
+
+class OnDiskBlockCreateTest: public Test {
+public:
+  OnDiskBlockCreateTest()
+  // Don't create the temp file yet (therefore pass false to the TempFile constructor)
+  : dir(),
+    key(Key::FromString("1491BB4932A389EE14BC7090AC772972")),
+    file(dir.path() / key.ToString(), false) {
+  }
+  TempDir dir;
+  Key key;
+  TempFile file;
+};
+
+TEST_F(OnDiskBlockCreateTest, CreatingBlockCreatesFile) {
+  EXPECT_FALSE(bf::exists(file.path()));
+
+  auto block = OnDiskBlock::CreateOnDisk(dir.path(), key, Data(0));
+
+  EXPECT_TRUE(bf::exists(file.path()));
+  EXPECT_TRUE(bf::is_regular_file(file.path()));
+}
+
+TEST_F(OnDiskBlockCreateTest, CreatingExistingBlockReturnsNull) {
+  auto block1 = OnDiskBlock::CreateOnDisk(dir.path(), key, Data(0));
+  auto block2 = OnDiskBlock::CreateOnDisk(dir.path(), key, Data(0));
+  EXPECT_TRUE((bool)block1);
+  EXPECT_FALSE((bool)block2);
+}
+
+class OnDiskBlockCreateSizeTest: public OnDiskBlockCreateTest, public WithParamInterface<size_t> {
+public:
+  unique_ref<OnDiskBlock> block;
+  Data ZEROES;
+
+  OnDiskBlockCreateSizeTest():
+    block(OnDiskBlock::CreateOnDisk(dir.path(), key, std::move(Data(GetParam()).FillWithZeroes())).value()),
+    ZEROES(block->size())
+  {
+    ZEROES.FillWithZeroes();
+  }
+};
+INSTANTIATE_TEST_CASE_P(OnDiskBlockCreateSizeTest, OnDiskBlockCreateSizeTest, Values(0, 1, 5, 1024, 10*1024*1024));
+
+TEST_P(OnDiskBlockCreateSizeTest, OnDiskSizeIsCorrect) {
+  Data fileContent = Data::LoadFromFile(file.path()).value();
+  EXPECT_EQ(GetParam(), fileContent.size());
+}
+
+TEST_P(OnDiskBlockCreateSizeTest, OnDiskBlockIsZeroedOut) {
+  Data fileContent = Data::LoadFromFile(file.path()).value();
+  EXPECT_EQ(ZEROES, fileContent);
+}
+
+// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface.
+// Here, we create it using OnDiskBlock::CreateOnDisk()
+TEST_P(OnDiskBlockCreateSizeTest, InMemorySizeIsCorrect) {
+  EXPECT_EQ(GetParam(), block->size());
+}
+
+// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface.
+// Here, we create it using OnDiskBlock::CreateOnDisk()
+TEST_P(OnDiskBlockCreateSizeTest, InMemoryBlockIsZeroedOut) {
+  EXPECT_EQ(0, std::memcmp(ZEROES.data(), block->data(), block->size()));
+}

test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp

@@ -0,0 +1,115 @@
+#include "../../../../implementations/ondisk/OnDiskBlock.h"
+#include <messmer/cpp-utils/data/DataFixture.h>
+#include "google/gtest/gtest.h"
+
+#include <messmer/cpp-utils/tempfile/TempFile.h>
+#include <messmer/cpp-utils/tempfile/TempDir.h>
+
+using ::testing::Test;
+using ::testing::WithParamInterface;
+using ::testing::Values;
+
+using cpputils::Data;
+using cpputils::DataFixture;
+using cpputils::TempFile;
+using cpputils::TempDir;
+using cpputils::unique_ref;
+
+using namespace blockstore;
+using namespace blockstore::ondisk;
+
+namespace bf = boost::filesystem;
+
+class OnDiskBlockFlushTest: public Test, public WithParamInterface<size_t> {
+public:
+  OnDiskBlockFlushTest()
+  // Don't create the temp file yet (therefore pass false to the TempFile constructor)
+  : dir(),
+    key(Key::FromString("1491BB4932A389EE14BC7090AC772972")),
+    file(dir.path() / key.ToString(), false),
+    randomData(DataFixture::generate(GetParam())) {
+  }
+  TempDir dir;
+  Key key;
+  TempFile file;
+
+  Data randomData;
+
+  unique_ref<OnDiskBlock> CreateBlockAndLoadItFromDisk() {
+    {
+      OnDiskBlock::CreateOnDisk(dir.path(), key, randomData.copy()).value();
+    }
+    return OnDiskBlock::LoadFromDisk(dir.path(), key).value();
+  }
+
+  unique_ref<OnDiskBlock> CreateBlock() {
+    return OnDiskBlock::CreateOnDisk(dir.path(), key, randomData.copy()).value();
+  }
+
+  void WriteDataToBlock(const unique_ref<OnDiskBlock> &block) {
+    block->write(randomData.data(), 0, randomData.size());
+  }
+
+  void EXPECT_BLOCK_DATA_CORRECT(const unique_ref<OnDiskBlock> &block) {
+    EXPECT_EQ(randomData.size(), block->size());
+    EXPECT_EQ(0, std::memcmp(randomData.data(), block->data(), randomData.size()));
+  }
+
+  void EXPECT_STORED_FILE_DATA_CORRECT() {
+    Data actual = Data::LoadFromFile(file.path()).value();
+    EXPECT_EQ(randomData, actual);
+  }
+};
+INSTANTIATE_TEST_CASE_P(OnDiskBlockFlushTest, OnDiskBlockFlushTest, Values((size_t)0, (size_t)1, (size_t)1024, (size_t)4096, (size_t)10*1024*1024));
+
+// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface.
+// Here, we create it using OnDiskBlock::CreateOnDisk()
+TEST_P(OnDiskBlockFlushTest, AfterCreate_FlushingDoesntChangeBlock) {
+  auto block =  CreateBlock();
+  WriteDataToBlock(block);
+
+  EXPECT_BLOCK_DATA_CORRECT(block);
+}
+
+// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface.
+// Here, we create it using OnDiskBlock::CreateOnDisk() / OnDiskBlock::LoadFromDisk()
+TEST_P(OnDiskBlockFlushTest, AfterLoad_FlushingDoesntChangeBlock) {
+  auto block =  CreateBlockAndLoadItFromDisk();
+  WriteDataToBlock(block);
+
+  EXPECT_BLOCK_DATA_CORRECT(block);
+}
+
+TEST_P(OnDiskBlockFlushTest, AfterCreate_FlushingWritesCorrectData) {
+  auto block = CreateBlock();
+  WriteDataToBlock(block);
+
+  EXPECT_STORED_FILE_DATA_CORRECT();
+}
+
+TEST_P(OnDiskBlockFlushTest, AfterLoad_FlushingWritesCorrectData) {
+  auto block = CreateBlockAndLoadItFromDisk();
+  WriteDataToBlock(block);
+
+  EXPECT_STORED_FILE_DATA_CORRECT();
+}
+
+// This test is also tested by OnDiskBlockStoreTest, but there it can only checks block content by loading it again.
+// Here, we check the content on disk.
+TEST_P(OnDiskBlockFlushTest, AfterCreate_FlushesWhenDestructed) {
+  {
+    auto block = CreateBlock();
+    WriteDataToBlock(block);
+  }
+  EXPECT_STORED_FILE_DATA_CORRECT();
+}
+
+// This test is also tested by OnDiskBlockStoreTest, but there it can only checks block content by loading it again.
+// Here, we check the content on disk.
+TEST_P(OnDiskBlockFlushTest, AfterLoad_FlushesWhenDestructed) {
+  {
+    auto block = CreateBlockAndLoadItFromDisk();
+    WriteDataToBlock(block);
+  }
+  EXPECT_STORED_FILE_DATA_CORRECT();
+}

test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp

@@ -0,0 +1,80 @@
+#include "../../../../implementations/ondisk/OnDiskBlock.h"
+#include <messmer/cpp-utils/data/DataFixture.h>
+#include "../../../../utils/FileDoesntExistException.h"
+#include "google/gtest/gtest.h"
+
+#include <messmer/cpp-utils/data/Data.h>
+#include <messmer/cpp-utils/tempfile/TempFile.h>
+#include <messmer/cpp-utils/tempfile/TempDir.h>
+#include <messmer/cpp-utils/pointer/unique_ref_boost_optional_gtest_workaround.h>
+#include <fstream>
+
+using ::testing::Test;
+using ::testing::WithParamInterface;
+using ::testing::Values;
+
+using std::ofstream;
+using std::ios;
+using cpputils::Data;
+using cpputils::DataFixture;
+using cpputils::TempFile;
+using cpputils::TempDir;
+using cpputils::unique_ref;
+
+using namespace blockstore;
+using namespace blockstore::ondisk;
+
+namespace bf = boost::filesystem;
+
+class OnDiskBlockLoadTest: public Test, public WithParamInterface<size_t> {
+public:
+  OnDiskBlockLoadTest():
+    dir(),
+    key(Key::FromString("1491BB4932A389EE14BC7090AC772972")),
+    file(dir.path() / key.ToString()) {
+  }
+  TempDir dir;
+  Key key;
+  TempFile file;
+
+  void SetFileSize(size_t size) {
+    Data data(size);
+    data.StoreToFile(file.path());
+  }
+
+  void StoreData(const Data &data) {
+    data.StoreToFile(file.path());
+  }
+
+  unique_ref<OnDiskBlock> LoadBlock() {
+    return OnDiskBlock::LoadFromDisk(dir.path(), key).value();
+  }
+
+  void EXPECT_BLOCK_DATA_EQ(const Data &expected, const OnDiskBlock &actual) {
+    EXPECT_EQ(expected.size(), actual.size());
+    EXPECT_EQ(0, std::memcmp(expected.data(), actual.data(), expected.size()));
+  }
+};
+INSTANTIATE_TEST_CASE_P(OnDiskBlockLoadTest, OnDiskBlockLoadTest, Values(0, 1, 5, 1024, 10*1024*1024));
+
+TEST_P(OnDiskBlockLoadTest, FileSizeIsCorrect) {
+  SetFileSize(GetParam());
+
+  auto block = LoadBlock();
+
+  EXPECT_EQ(GetParam(), block->size());
+}
+
+TEST_P(OnDiskBlockLoadTest, LoadedDataIsCorrect) {
+  Data randomData = DataFixture::generate(GetParam());
+  StoreData(randomData);
+
+  auto block = LoadBlock();
+
+  EXPECT_BLOCK_DATA_EQ(randomData, *block);
+}
+
+TEST_F(OnDiskBlockLoadTest, LoadNotExistingBlock) {
+  Key key2 = Key::FromString("272EE5517627CFA147A971A8E6E747E0");
+  EXPECT_EQ(boost::none, OnDiskBlock::LoadFromDisk(dir.path(), key2));
+}

test/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreTest.cpp

@@ -0,0 +1,23 @@
+#include "../../../implementations/parallelaccess/ParallelAccessBlockStore.h"
+#include "../../../implementations/testfake/FakeBlockStore.h"
+#include "../../testutils/BlockStoreTest.h"
+#include "google/gtest/gtest.h"
+
+
+using blockstore::BlockStore;
+using blockstore::parallelaccess::ParallelAccessBlockStore;
+using blockstore::testfake::FakeBlockStore;
+
+using cpputils::make_unique_ref;
+using cpputils::unique_ref;
+
+class ParallelAccessBlockStoreTestFixture: public BlockStoreTestFixture {
+public:
+  unique_ref<BlockStore> createBlockStore() override {
+    return make_unique_ref<ParallelAccessBlockStore>(make_unique_ref<FakeBlockStore>());
+  }
+};
+
+INSTANTIATE_TYPED_TEST_CASE_P(ParallelAccess, BlockStoreTest, ParallelAccessBlockStoreTestFixture);
+
+//TODO Add specific tests ensuring that loading the same block twice doesn't load it twice from the underlying blockstore

test/blockstore/implementations/testfake/TestFakeBlockStoreTest.cpp

@@ -0,0 +1,30 @@
+#include "../../../implementations/testfake/FakeBlock.h"
+#include "../../../implementations/testfake/FakeBlockStore.h"
+#include "../../testutils/BlockStoreTest.h"
+#include "../../testutils/BlockStoreWithRandomKeysTest.h"
+#include "google/gtest/gtest.h"
+
+
+using blockstore::BlockStore;
+using blockstore::BlockStoreWithRandomKeys;
+using blockstore::testfake::FakeBlockStore;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+
+class FakeBlockStoreTestFixture: public BlockStoreTestFixture {
+public:
+  unique_ref<BlockStore> createBlockStore() override {
+    return make_unique_ref<FakeBlockStore>();
+  }
+};
+
+INSTANTIATE_TYPED_TEST_CASE_P(TestFake, BlockStoreTest, FakeBlockStoreTestFixture);
+
+class FakeBlockStoreWithRandomKeysTestFixture: public BlockStoreWithRandomKeysTestFixture {
+public:
+  unique_ref<BlockStoreWithRandomKeys> createBlockStore() override {
+    return make_unique_ref<FakeBlockStore>();
+  }
+};
+
+INSTANTIATE_TYPED_TEST_CASE_P(TestFake, BlockStoreWithRandomKeysTest, FakeBlockStoreWithRandomKeysTestFixture);

test/blockstore/interface/BlockStoreTest.cpp

@@ -0,0 +1,4 @@
+/*
+ * Tests that the header can be included without needing additional header includes as dependencies.
+ */
+#include "../../interface/BlockStore.h"

test/blockstore/interface/BlockTest.cpp

@@ -0,0 +1,4 @@
+/*
+ * Tests that the header can be included without needing additional header includes as dependencies.
+ */
+#include "../../interface/Block.h"

test/blockstore/interface/helpers/BlockStoreWithRandomKeysTest.cpp

@@ -0,0 +1,143 @@
+#include "../../../interface/helpers/BlockStoreWithRandomKeys.h"
+#include "google/gtest/gtest.h"
+#include "google/gmock/gmock.h"
+#include <messmer/cpp-utils/data/DataFixture.h>
+
+using ::testing::Test;
+using ::testing::_;
+using ::testing::Return;
+using ::testing::Invoke;
+using ::testing::Eq;
+using ::testing::ByRef;
+
+using std::string;
+using cpputils::Data;
+using cpputils::DataFixture;
+using cpputils::unique_ref;
+using boost::optional;
+
+using namespace blockstore;
+
+class BlockStoreWithRandomKeysMock: public BlockStoreWithRandomKeys {
+public:
+  optional<unique_ref<Block>> tryCreate(const Key &key, Data data) {
+    return cpputils::nullcheck(std::unique_ptr<Block>(do_create(key, data)));
+  }
+  MOCK_METHOD2(do_create, Block*(const Key &, const Data &data));
+  optional<unique_ref<Block>> load(const Key &key) {
+    return cpputils::nullcheck(std::unique_ptr<Block>(do_load(key)));
+  }
+  MOCK_METHOD1(do_load, Block*(const Key &));
+  void remove(unique_ref<Block> block) {UNUSED(block);}
+  MOCK_CONST_METHOD0(numBlocks, uint64_t());
+};
+
+class BlockMock: public Block {
+public:
+  BlockMock(): Block(cpputils::Random::PseudoRandom().getFixedSize<Key::BINARY_LENGTH>()) {}
+  MOCK_CONST_METHOD0(data, const void*());
+  MOCK_METHOD3(write, void(const void*, uint64_t, uint64_t));
+  MOCK_METHOD0(flush, void());
+  MOCK_CONST_METHOD0(size, size_t());
+  MOCK_METHOD1(resize, void(size_t));
+  MOCK_CONST_METHOD0(key, const Key&());
+};
+
+class BlockStoreWithRandomKeysTest: public Test {
+public:
+  BlockStoreWithRandomKeysTest() :blockStoreMock(), blockStore(blockStoreMock),
+                                  key(Key::FromString("1491BB4932A389EE14BC7090AC772972")) {}
+
+  BlockStoreWithRandomKeysMock blockStoreMock;
+  BlockStore &blockStore;
+  const blockstore::Key key;
+
+  Data createDataWithSize(size_t size) {
+	Data fixture(DataFixture::generate(size));
+	Data data(size);
+	std::memcpy(data.data(), fixture.data(), size);
+	return data;
+  }
+};
+
+TEST_F(BlockStoreWithRandomKeysTest, DataIsPassedThrough0) {
+  Data data = createDataWithSize(0);
+  EXPECT_CALL(blockStoreMock, do_create(_, Eq(ByRef(data)))).WillOnce(Return(new BlockMock));
+  blockStore.create(data);
+}
+
+TEST_F(BlockStoreWithRandomKeysTest, DataIsPassedThrough1) {
+  Data data = createDataWithSize(1);
+  EXPECT_CALL(blockStoreMock, do_create(_, Eq(ByRef(data)))).WillOnce(Return(new BlockMock));
+  blockStore.create(data);
+}
+
+TEST_F(BlockStoreWithRandomKeysTest, DataIsPassedThrough1024) {
+  Data data = createDataWithSize(1024);
+  EXPECT_CALL(blockStoreMock, do_create(_, Eq(ByRef(data)))).WillOnce(Return(new BlockMock));
+  blockStore.create(data);
+}
+
+TEST_F(BlockStoreWithRandomKeysTest, KeyHasCorrectSize) {
+  EXPECT_CALL(blockStoreMock, do_create(_, _)).WillOnce(Invoke([](const Key &key, const Data &) {
+    EXPECT_EQ(Key::STRING_LENGTH, key.ToString().size());
+    return new BlockMock;
+  }));
+
+  blockStore.create(createDataWithSize(1024));
+}
+
+TEST_F(BlockStoreWithRandomKeysTest, TwoBlocksGetDifferentKeys) {
+  Key first_key = key;
+  EXPECT_CALL(blockStoreMock, do_create(_, _))
+      .WillOnce(Invoke([&first_key](const Key &key, const Data &) {
+        first_key = key;
+        return new BlockMock;
+      }))
+      .WillOnce(Invoke([&first_key](const Key &key, const Data &) {
+        EXPECT_NE(first_key, key);
+        return new BlockMock;
+      }));
+
+  Data data = createDataWithSize(1024);
+  blockStore.create(data);
+  blockStore.create(data);
+}
+
+TEST_F(BlockStoreWithRandomKeysTest, WillTryADifferentKeyIfKeyAlreadyExists) {
+  Key first_key = key;
+  Data data = createDataWithSize(1024);
+  EXPECT_CALL(blockStoreMock, do_create(_, Eq(ByRef(data))))
+      .WillOnce(Invoke([&first_key](const Key &key, const Data &) {
+        first_key = key;
+        return nullptr;
+      }))
+	  //TODO Check that this test case fails when the second do_create call gets different data
+      .WillOnce(Invoke([&first_key](const Key &key, const Data &) {
+        EXPECT_NE(first_key, key);
+        return new BlockMock;
+      }));
+
+  blockStore.create(data);
+}
+
+TEST_F(BlockStoreWithRandomKeysTest, WillTryADifferentKeyIfKeyAlreadyExistsTwoTimes) {
+  Key first_key = key;
+  Data data = createDataWithSize(1024);
+  EXPECT_CALL(blockStoreMock, do_create(_, Eq(ByRef(data))))
+      .WillOnce(Invoke([&first_key](const Key &key, const Data &) {
+        first_key = key;
+        return nullptr;
+      }))
+	  //TODO Check that this test case fails when the second/third do_create calls get different data
+      .WillOnce(Invoke([&first_key](const Key &key, const Data &) {
+        first_key = key;
+        return nullptr;
+      }))
+      .WillOnce(Invoke([&first_key](const Key &key, const Data &) {
+        EXPECT_NE(first_key, key);
+        return new BlockMock;
+      }));
+
+  blockStore.create(data);
+}

test/blockstore/testutils/BlockStoreTest.h

@@ -0,0 +1,148 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_TESTUTILS_BLOCKSTORETEST_H_
+#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_TESTUTILS_BLOCKSTORETEST_H_
+
+#include "google/gtest/gtest.h"
+
+#include "../../interface/BlockStore.h"
+
+class BlockStoreTestFixture {
+public:
+  virtual ~BlockStoreTestFixture() {}
+  virtual cpputils::unique_ref<blockstore::BlockStore> createBlockStore() = 0;
+};
+
+template<class ConcreteBlockStoreTestFixture>
+class BlockStoreTest: public ::testing::Test {
+public:
+  BlockStoreTest() :fixture() {}
+
+  BOOST_STATIC_ASSERT_MSG(
+    (std::is_base_of<BlockStoreTestFixture, ConcreteBlockStoreTestFixture>::value),
+    "Given test fixture for instantiating the (type parameterized) BlockStoreTest must inherit from BlockStoreTestFixture"
+  );
+
+  ConcreteBlockStoreTestFixture fixture;
+};
+
+TYPED_TEST_CASE_P(BlockStoreTest);
+
+TYPED_TEST_P(BlockStoreTest, TwoCreatedBlocksHaveDifferentKeys) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto block1 = blockStore->create(cpputils::Data(1024));
+  auto block2 = blockStore->create(cpputils::Data(1024));
+  EXPECT_NE(block1->key(), block2->key());
+}
+
+TYPED_TEST_P(BlockStoreTest, BlockIsNotLoadableAfterDeleting) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto blockkey = blockStore->create(cpputils::Data(1024))->key();
+  auto block = blockStore->load(blockkey);
+  EXPECT_NE(boost::none, block);
+  blockStore->remove(std::move(*block));
+  EXPECT_EQ(boost::none, blockStore->load(blockkey));
+}
+
+TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectOnEmptyBlockstore) {
+  auto blockStore = this->fixture.createBlockStore();
+  EXPECT_EQ(0, blockStore->numBlocks());
+}
+
+TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingOneBlock) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto block = blockStore->create(cpputils::Data(1));
+  EXPECT_EQ(1, blockStore->numBlocks());
+}
+
+TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingOneBlock_AfterClosingBlock) {
+  auto blockStore = this->fixture.createBlockStore();
+  blockStore->create(cpputils::Data(1));
+  EXPECT_EQ(1, blockStore->numBlocks());
+}
+
+TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterRemovingTheLastBlock) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto block = blockStore->create(cpputils::Data(1));
+  blockStore->remove(std::move(block));
+  EXPECT_EQ(0, blockStore->numBlocks());
+}
+
+TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto block1 = blockStore->create(cpputils::Data(1));
+  auto block2 = blockStore->create(cpputils::Data(0));
+  EXPECT_EQ(2, blockStore->numBlocks());
+}
+
+TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingFirstBlock) {
+  auto blockStore = this->fixture.createBlockStore();
+  blockStore->create(cpputils::Data(1));
+  auto block2 = blockStore->create(cpputils::Data(0));
+  EXPECT_EQ(2, blockStore->numBlocks());
+}
+
+TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingSecondBlock) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto block1 = blockStore->create(cpputils::Data(1));
+  blockStore->create(cpputils::Data(0));
+  EXPECT_EQ(2, blockStore->numBlocks());
+}
+
+TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingBothBlocks) {
+  auto blockStore = this->fixture.createBlockStore();
+  blockStore->create(cpputils::Data(1));
+  blockStore->create(cpputils::Data(0));
+  EXPECT_EQ(2, blockStore->numBlocks());
+}
+
+TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterRemovingABlock) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto block = blockStore->create(cpputils::Data(1));
+  blockStore->create(cpputils::Data(1));
+  blockStore->remove(std::move(block));
+  EXPECT_EQ(1, blockStore->numBlocks());
+}
+
+TYPED_TEST_P(BlockStoreTest, CanRemoveModifiedBlock) {
+    auto blockStore = this->fixture.createBlockStore();
+    auto block = blockStore->create(cpputils::Data(5));
+    block->write("data", 0, 4);
+    blockStore->remove(std::move(block));
+    EXPECT_EQ(0, blockStore->numBlocks());
+}
+
+#include "BlockStoreTest_Size.h"
+#include "BlockStoreTest_Data.h"
+
+
+REGISTER_TYPED_TEST_CASE_P(BlockStoreTest,
+    CreatedBlockHasCorrectSize,
+    LoadingUnchangedBlockHasCorrectSize,
+    CreatedBlockData,
+    LoadingUnchangedBlockData,
+    LoadedBlockIsCorrect,
+//    LoadedBlockIsCorrectWhenLoadedDirectlyAfterFlushing,
+    AfterCreate_FlushingDoesntChangeBlock,
+    AfterLoad_FlushingDoesntChangeBlock,
+    AfterCreate_FlushesWhenDestructed,
+    AfterLoad_FlushesWhenDestructed,
+    LoadNonExistingBlock,
+    TwoCreatedBlocksHaveDifferentKeys,
+    BlockIsNotLoadableAfterDeleting,
+    NumBlocksIsCorrectOnEmptyBlockstore,
+    NumBlocksIsCorrectAfterAddingOneBlock,
+    NumBlocksIsCorrectAfterAddingOneBlock_AfterClosingBlock,
+    NumBlocksIsCorrectAfterRemovingTheLastBlock,
+    NumBlocksIsCorrectAfterAddingTwoBlocks,
+    NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingFirstBlock,
+    NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingSecondBlock,
+    NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingBothBlocks,
+    NumBlocksIsCorrectAfterRemovingABlock,
+    WriteAndReadImmediately,
+    WriteAndReadAfterLoading,
+    OverwriteAndRead,
+    CanRemoveModifiedBlock
+);
+
+
+#endif

test/blockstore/testutils/BlockStoreTest_Data.h

@@ -0,0 +1,107 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTORETEST_DATA_H_
+#define MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTORETEST_DATA_H_
+
+// This file is meant to be included by BlockStoreTest.h only
+
+struct DataRange {
+  size_t blocksize;
+  off_t offset;
+  size_t count;
+};
+
+class BlockStoreDataParametrizedTest {
+public:
+  BlockStoreDataParametrizedTest(cpputils::unique_ref<blockstore::BlockStore> blockStore_, const DataRange &testData_)
+    : blockStore(std::move(blockStore_)),
+      testData(testData_),
+      foregroundData(cpputils::DataFixture::generate(testData.count, 0)),
+      backgroundData(cpputils::DataFixture::generate(testData.blocksize, 1)) {
+  }
+
+  void TestWriteAndReadImmediately() {
+    auto block = blockStore->create(cpputils::Data(testData.blocksize).FillWithZeroes());
+    block->write(foregroundData.data(), testData.offset, testData.count);
+
+    EXPECT_DATA_READS_AS(foregroundData, *block, testData.offset, testData.count);
+    EXPECT_DATA_IS_ZEROES_OUTSIDE_OF(*block, testData.offset, testData.count);
+  }
+
+  void TestWriteAndReadAfterLoading() {
+    blockstore::Key key = CreateBlockWriteToItAndReturnKey(foregroundData);
+
+    auto loaded_block = blockStore->load(key).value();
+    EXPECT_DATA_READS_AS(foregroundData, *loaded_block, testData.offset, testData.count);
+    EXPECT_DATA_IS_ZEROES_OUTSIDE_OF(*loaded_block, testData.offset, testData.count);
+  }
+
+  void TestOverwriteAndRead() {
+    auto block = blockStore->create(cpputils::Data(testData.blocksize));
+    block->write(backgroundData.data(), 0, testData.blocksize);
+    block->write(foregroundData.data(), testData.offset, testData.count);
+    EXPECT_DATA_READS_AS(foregroundData, *block, testData.offset, testData.count);
+    EXPECT_DATA_READS_AS_OUTSIDE_OF(backgroundData, *block, testData.offset, testData.count);
+  }
+
+private:
+  cpputils::unique_ref<blockstore::BlockStore> blockStore;
+  DataRange testData;
+  cpputils::Data foregroundData;
+  cpputils::Data backgroundData;
+
+  blockstore::Key CreateBlockWriteToItAndReturnKey(const cpputils::Data &to_write) {
+    auto newblock = blockStore->create(cpputils::Data(testData.blocksize).FillWithZeroes());
+
+    newblock->write(to_write.data(), testData.offset, testData.count);
+    return newblock->key();
+  }
+
+  void EXPECT_DATA_READS_AS(const cpputils::Data &expected, const blockstore::Block &block, off_t offset, size_t count) {
+    cpputils::Data read(count);
+    std::memcpy(read.data(), (uint8_t*)block.data() + offset, count);
+    EXPECT_EQ(expected, read);
+  }
+
+  void EXPECT_DATA_READS_AS_OUTSIDE_OF(const cpputils::Data &expected, const blockstore::Block &block, off_t start, size_t count) {
+    cpputils::Data begin(start);
+    cpputils::Data end(testData.blocksize - count - start);
+
+    std::memcpy(begin.data(), expected.data(), start);
+    std::memcpy(end.data(), (uint8_t*)expected.data()+start+count, end.size());
+
+    EXPECT_DATA_READS_AS(begin, block, 0, start);
+    EXPECT_DATA_READS_AS(end, block, start + count, end.size());
+  }
+
+  void EXPECT_DATA_IS_ZEROES_OUTSIDE_OF(const blockstore::Block &block, off_t start, size_t count) {
+    cpputils::Data ZEROES(testData.blocksize);
+    ZEROES.FillWithZeroes();
+    EXPECT_DATA_READS_AS_OUTSIDE_OF(ZEROES, block, start, count);
+  }
+};
+
+inline std::vector<DataRange> DATA_RANGES() {
+  return {
+          DataRange{1024, 0, 1024},     // full size leaf, access beginning to end
+          DataRange{1024, 100, 1024 - 200}, // full size leaf, access middle to middle
+          DataRange{1024, 0, 1024 - 100}, // full size leaf, access beginning to middle
+          DataRange{1024, 100, 1024 - 100}, // full size leaf, access middle to end
+          DataRange{1024 - 100, 0, 1024 - 100}, // non-full size leaf, access beginning to end
+          DataRange{1024 - 100, 100, 1024 - 300}, // non-full size leaf, access middle to middle
+          DataRange{1024 - 100, 0, 1024 - 200}, // non-full size leaf, access beginning to middle
+          DataRange{1024 - 100, 100, 1024 - 200}  // non-full size leaf, access middle to end
+  };
+};
+#define TYPED_TEST_P_FOR_ALL_DATA_RANGES(TestName)                                   \
+  TYPED_TEST_P(BlockStoreTest, TestName) {                                           \
+    for (auto dataRange: DATA_RANGES()) {                                            \
+      BlockStoreDataParametrizedTest(this->fixture.createBlockStore(), dataRange)    \
+        .Test##TestName();                                                           \
+    }                                                                                \
+  }
+
+TYPED_TEST_P_FOR_ALL_DATA_RANGES(WriteAndReadImmediately);
+TYPED_TEST_P_FOR_ALL_DATA_RANGES(WriteAndReadAfterLoading);
+TYPED_TEST_P_FOR_ALL_DATA_RANGES(OverwriteAndRead);
+
+#endif

test/blockstore/testutils/BlockStoreTest_Size.h

@@ -0,0 +1,164 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTORETEST_SIZE_H_
+#define MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTORETEST_SIZE_H_
+
+// This file is meant to be included by BlockStoreTest.h only
+
+#include <messmer/cpp-utils/data/Data.h>
+#include <messmer/cpp-utils/data/DataFixture.h>
+
+class BlockStoreSizeParameterizedTest {
+public:
+  BlockStoreSizeParameterizedTest(cpputils::unique_ref<blockstore::BlockStore> blockStore_, size_t size_): blockStore(std::move(blockStore_)), size(size_) {}
+
+  void TestCreatedBlockHasCorrectSize() {
+    auto block = CreateBlock();
+    EXPECT_EQ(size, block->size());
+  }
+
+  void TestLoadingUnchangedBlockHasCorrectSize() {
+    blockstore::Key key = CreateBlock()->key();
+    auto loaded_block = blockStore->load(key).value();
+    EXPECT_EQ(size, loaded_block->size());
+  }
+
+  void TestCreatedBlockData() {
+    cpputils::Data data = cpputils::DataFixture::generate(size);
+    auto block = blockStore->create(data);
+    EXPECT_EQ(0, std::memcmp(data.data(), block->data(), size));
+  }
+
+  void TestLoadingUnchangedBlockData() {
+    cpputils::Data data = cpputils::DataFixture::generate(size);
+    blockstore::Key key = blockStore->create(data)->key();
+    auto loaded_block = blockStore->load(key).value();
+    EXPECT_EQ(0, std::memcmp(data.data(), loaded_block->data(), size));
+  }
+
+  void TestLoadedBlockIsCorrect() {
+    cpputils::Data randomData = cpputils::DataFixture::generate(size);
+    auto loaded_block = StoreDataToBlockAndLoadIt(randomData);
+    EXPECT_EQ(size, loaded_block->size());
+    EXPECT_EQ(0, std::memcmp(randomData.data(), loaded_block->data(), size));
+  }
+
+  void TestLoadedBlockIsCorrectWhenLoadedDirectlyAfterFlushing() {
+    cpputils::Data randomData = cpputils::DataFixture::generate(size);
+    auto loaded_block = StoreDataToBlockAndLoadItDirectlyAfterFlushing(randomData);
+    EXPECT_EQ(size, loaded_block->size());
+    EXPECT_EQ(0, std::memcmp(randomData.data(), loaded_block->data(), size));
+  }
+
+  void TestAfterCreate_FlushingDoesntChangeBlock() {
+    cpputils::Data randomData = cpputils::DataFixture::generate(size);
+    auto block =  CreateBlock();
+    WriteDataToBlock(block.get(), randomData);
+    block->flush();
+
+    EXPECT_BLOCK_DATA_CORRECT(*block, randomData);
+  }
+
+  void TestAfterLoad_FlushingDoesntChangeBlock() {
+    cpputils::Data randomData = cpputils::DataFixture::generate(size);
+    auto block =  CreateBlockAndLoadIt();
+    WriteDataToBlock(block.get(), randomData);
+    block->flush();
+
+    EXPECT_BLOCK_DATA_CORRECT(*block, randomData);
+  }
+
+  void TestAfterCreate_FlushesWhenDestructed() {
+    cpputils::Data randomData = cpputils::DataFixture::generate(size);
+    blockstore::Key key = blockstore::Key::Null();
+    {
+      auto block = blockStore->create(cpputils::Data(size));
+      key = block->key();
+      WriteDataToBlock(block.get(), randomData);
+    }
+    auto loaded_block = blockStore->load(key).value();
+    EXPECT_BLOCK_DATA_CORRECT(*loaded_block, randomData);
+  }
+
+  void TestAfterLoad_FlushesWhenDestructed() {
+    cpputils::Data randomData = cpputils::DataFixture::generate(size);
+    blockstore::Key key = blockstore::Key::Null();
+    {
+      key = CreateBlock()->key();
+      auto block = blockStore->load(key).value();
+      WriteDataToBlock(block.get(), randomData);
+    }
+    auto loaded_block = blockStore->load(key).value();
+    EXPECT_BLOCK_DATA_CORRECT(*loaded_block, randomData);
+  }
+
+  void TestLoadNonExistingBlock() {
+    EXPECT_EQ(boost::none, blockStore->load(key));
+  }
+
+private:
+  const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
+  cpputils::unique_ref<blockstore::BlockStore> blockStore;
+  size_t size;
+
+  cpputils::Data ZEROES(size_t size) {
+    cpputils::Data ZEROES(size);
+    ZEROES.FillWithZeroes();
+    return ZEROES;
+  }
+
+  cpputils::unique_ref<blockstore::Block> StoreDataToBlockAndLoadIt(const cpputils::Data &data) {
+    blockstore::Key key = StoreDataToBlockAndGetKey(data);
+    return blockStore->load(key).value();
+  }
+
+  blockstore::Key StoreDataToBlockAndGetKey(const cpputils::Data &data) {
+    return blockStore->create(data)->key();
+  }
+
+  cpputils::unique_ref<blockstore::Block> StoreDataToBlockAndLoadItDirectlyAfterFlushing(const cpputils::Data &data) {
+    auto block = blockStore->create(data);
+    block->flush();
+    return blockStore->load(block->key()).value();
+  }
+
+  cpputils::unique_ref<blockstore::Block> CreateBlockAndLoadIt() {
+    blockstore::Key key = CreateBlock()->key();
+    return blockStore->load(key).value();
+  }
+
+  cpputils::unique_ref<blockstore::Block> CreateBlock() {
+    return blockStore->create(cpputils::Data(size));
+  }
+
+  void WriteDataToBlock(blockstore::Block *block, const cpputils::Data &randomData) {
+    block->write(randomData.data(), 0, randomData.size());
+  }
+
+  void EXPECT_BLOCK_DATA_CORRECT(const blockstore::Block &block, const cpputils::Data &randomData) {
+    EXPECT_EQ(randomData.size(), block.size());
+    EXPECT_EQ(0, std::memcmp(randomData.data(), block.data(), randomData.size()));
+  }
+};
+
+constexpr std::initializer_list<size_t> SIZES = {0, 1, 1024, 4096, 10*1024*1024};
+#define TYPED_TEST_P_FOR_ALL_SIZES(TestName)                                    \
+  TYPED_TEST_P(BlockStoreTest, TestName) {                                      \
+    for (auto size: SIZES) {                                                    \
+      BlockStoreSizeParameterizedTest(this->fixture.createBlockStore(), size)   \
+        .Test##TestName();                                                      \
+    }                                                                           \
+  }                                                                             \
+
+TYPED_TEST_P_FOR_ALL_SIZES(CreatedBlockHasCorrectSize);
+TYPED_TEST_P_FOR_ALL_SIZES(LoadingUnchangedBlockHasCorrectSize);
+TYPED_TEST_P_FOR_ALL_SIZES(CreatedBlockData);
+TYPED_TEST_P_FOR_ALL_SIZES(LoadingUnchangedBlockData);
+TYPED_TEST_P_FOR_ALL_SIZES(LoadedBlockIsCorrect);
+//TYPED_TEST_P_FOR_ALL_SIZES(LoadedBlockIsCorrectWhenLoadedDirectlyAfterFlushing);
+TYPED_TEST_P_FOR_ALL_SIZES(AfterCreate_FlushingDoesntChangeBlock);
+TYPED_TEST_P_FOR_ALL_SIZES(AfterLoad_FlushingDoesntChangeBlock);
+TYPED_TEST_P_FOR_ALL_SIZES(AfterCreate_FlushesWhenDestructed);
+TYPED_TEST_P_FOR_ALL_SIZES(AfterLoad_FlushesWhenDestructed);
+TYPED_TEST_P_FOR_ALL_SIZES(LoadNonExistingBlock);
+
+#endif

test/blockstore/testutils/BlockStoreWithRandomKeysTest.h

@@ -0,0 +1,88 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTOREWITHRANDOMKEYSTEST_H_
+#define MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTOREWITHRANDOMKEYSTEST_H_
+
+#include <google/gtest/gtest.h>
+
+#include "../../interface/BlockStore.h"
+
+class BlockStoreWithRandomKeysTestFixture {
+public:
+  virtual ~BlockStoreWithRandomKeysTestFixture() {}
+  virtual cpputils::unique_ref<blockstore::BlockStoreWithRandomKeys> createBlockStore() = 0;
+};
+
+template<class ConcreteBlockStoreWithRandomKeysTestFixture>
+class BlockStoreWithRandomKeysTest: public ::testing::Test {
+public:
+  BlockStoreWithRandomKeysTest(): fixture() {}
+
+  BOOST_STATIC_ASSERT_MSG(
+    (std::is_base_of<BlockStoreWithRandomKeysTestFixture, ConcreteBlockStoreWithRandomKeysTestFixture>::value),
+    "Given test fixture for instantiating the (type parameterized) BlockStoreWithRandomKeysTest must inherit from BlockStoreWithRandomKeysTestFixture"
+  );
+
+  blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
+
+  const std::vector<size_t> SIZES = {0, 1, 1024, 4096, 10*1024*1024};
+
+  ConcreteBlockStoreWithRandomKeysTestFixture fixture;
+};
+
+TYPED_TEST_CASE_P(BlockStoreWithRandomKeysTest);
+
+TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndSameSize) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto block = blockStore->tryCreate(this->key, cpputils::Data(1024));
+  (*block)->flush(); //TODO Ideally, flush shouldn't be necessary here.
+  auto block2 = blockStore->tryCreate(this->key, cpputils::Data(1024));
+  EXPECT_NE(boost::none, block);
+  EXPECT_EQ(boost::none, block2);
+}
+
+TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndDifferentSize) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto block = blockStore->tryCreate(this->key, cpputils::Data(1024));
+  (*block)->flush(); //TODO Ideally, flush shouldn't be necessary here.
+  auto block2 = blockStore->tryCreate(this->key, cpputils::Data(4096));
+  EXPECT_NE(boost::none, block);
+  EXPECT_EQ(boost::none, block2);
+}
+
+TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndFirstNullSize) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto block = blockStore->tryCreate(this->key, cpputils::Data(0));
+  (*block)->flush(); //TODO Ideally, flush shouldn't be necessary here.
+  auto block2 = blockStore->tryCreate(this->key, cpputils::Data(1024));
+  EXPECT_NE(boost::none, block);
+  EXPECT_EQ(boost::none, block2);
+}
+
+TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndSecondNullSize) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto block = blockStore->tryCreate(this->key, cpputils::Data(1024));
+  (*block)->flush(); //TODO Ideally, flush shouldn't be necessary here.
+  auto block2 = blockStore->tryCreate(this->key, cpputils::Data(0));
+  EXPECT_NE(boost::none, block);
+  EXPECT_EQ(boost::none, block2);
+}
+
+TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndBothNullSize) {
+  auto blockStore = this->fixture.createBlockStore();
+  auto block = blockStore->tryCreate(this->key, cpputils::Data(0));
+  (*block)->flush(); //TODO Ideally, flush shouldn't be necessary here.
+  auto block2 = blockStore->tryCreate(this->key, cpputils::Data(0));
+  EXPECT_NE(boost::none, block);
+  EXPECT_EQ(boost::none, block2);
+}
+
+REGISTER_TYPED_TEST_CASE_P(BlockStoreWithRandomKeysTest,
+  CreateTwoBlocksWithSameKeyAndSameSize,
+  CreateTwoBlocksWithSameKeyAndDifferentSize,
+  CreateTwoBlocksWithSameKeyAndFirstNullSize,
+  CreateTwoBlocksWithSameKeyAndSecondNullSize,
+  CreateTwoBlocksWithSameKeyAndBothNullSize
+);
+
+
+#endif

test/blockstore/utils/BlockStoreUtilsTest.cpp

@@ -0,0 +1,113 @@
+#include "../../implementations/testfake/FakeBlockStore.h"
+#include <messmer/cpp-utils/data/DataFixture.h>
+#include "../../utils/BlockStoreUtils.h"
+#include "google/gtest/gtest.h"
+
+#include <memory>
+
+using ::testing::Test;
+using ::testing::WithParamInterface;
+using ::testing::Values;
+
+using cpputils::Data;
+using cpputils::DataFixture;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+
+using namespace blockstore;
+using namespace blockstore::utils;
+
+using blockstore::testfake::FakeBlockStore;
+
+class BlockStoreUtilsTest: public Test {
+public:
+  unsigned int SIZE = 1024 * 1024;
+  BlockStoreUtilsTest():
+    ZEROES(SIZE),
+    dataFixture(DataFixture::generate(SIZE)),
+    blockStore(make_unique_ref<FakeBlockStore>()) {
+    ZEROES.FillWithZeroes();
+  }
+
+  Data ZEROES;
+  Data dataFixture;
+  unique_ref<BlockStore> blockStore;
+};
+
+TEST_F(BlockStoreUtilsTest, FillWithZeroes) {
+  auto block = blockStore->create(Data(SIZE));
+  block->write(dataFixture.data(), 0, SIZE);
+  EXPECT_NE(0, std::memcmp(ZEROES.data(), block->data(), SIZE));
+  fillWithZeroes(block.get());
+  EXPECT_EQ(0, std::memcmp(ZEROES.data(), block->data(), SIZE));
+}
+
+class BlockStoreUtilsTest_CopyToNewBlock: public BlockStoreUtilsTest {};
+
+TEST_F(BlockStoreUtilsTest_CopyToNewBlock, CopyEmptyBlock) {
+  auto block = blockStore->create(Data(0));
+  auto block2 = copyToNewBlock(blockStore.get(), *block);
+
+  EXPECT_EQ(0u, block2->size());
+}
+
+TEST_F(BlockStoreUtilsTest_CopyToNewBlock, CopyZeroBlock) {
+  auto block = blockStore->create(ZEROES);
+  auto block2 = copyToNewBlock(blockStore.get(), *block);
+
+  EXPECT_EQ(SIZE, block2->size());
+  EXPECT_EQ(0, std::memcmp(ZEROES.data(), block2->data(), SIZE));
+}
+
+TEST_F(BlockStoreUtilsTest_CopyToNewBlock, CopyDataBlock) {
+  auto block = blockStore->create(Data(SIZE));
+  block->write(dataFixture.data(), 0, SIZE);
+  auto block2 = copyToNewBlock(blockStore.get(), *block);
+
+  EXPECT_EQ(SIZE, block2->size());
+  EXPECT_EQ(0, std::memcmp(dataFixture.data(), block2->data(), SIZE));
+}
+
+TEST_F(BlockStoreUtilsTest_CopyToNewBlock, OriginalBlockUnchanged) {
+  auto block = blockStore->create(Data(SIZE));
+  block->write(dataFixture.data(), 0, SIZE);
+  auto block2 = copyToNewBlock(blockStore.get(), *block);
+
+  EXPECT_EQ(SIZE, block->size());
+  EXPECT_EQ(0, std::memcmp(dataFixture.data(), block->data(), SIZE));
+}
+
+class BlockStoreUtilsTest_CopyToExistingBlock: public BlockStoreUtilsTest {};
+
+TEST_F(BlockStoreUtilsTest_CopyToExistingBlock, CopyEmptyBlock) {
+  auto block = blockStore->create(Data(0));
+  auto block2 = blockStore->create(Data(0));
+  copyTo(block2.get(), *block);
+}
+
+TEST_F(BlockStoreUtilsTest_CopyToExistingBlock, CopyZeroBlock) {
+  auto block = blockStore->create(ZEROES);
+  auto block2 = blockStore->create(Data(SIZE));
+  block2->write(dataFixture.data(), 0, SIZE);
+  copyTo(block2.get(), *block);
+
+  EXPECT_EQ(0, std::memcmp(ZEROES.data(), block2->data(), SIZE));
+}
+
+TEST_F(BlockStoreUtilsTest_CopyToExistingBlock, CopyDataBlock) {
+  auto block = blockStore->create(Data(SIZE));
+  block->write(dataFixture.data(), 0, SIZE);
+  auto block2 = blockStore->create(Data(SIZE));
+  copyTo(block2.get(), *block);
+
+  EXPECT_EQ(0, std::memcmp(dataFixture.data(), block2->data(), SIZE));
+}
+
+TEST_F(BlockStoreUtilsTest_CopyToExistingBlock, OriginalBlockUnchanged) {
+  auto block = blockStore->create(Data(SIZE));
+  block->write(dataFixture.data(), 0, SIZE);
+  auto block2 = blockStore->create(Data(SIZE));
+  copyTo(block2.get(), *block);
+
+  EXPECT_EQ(0, std::memcmp(dataFixture.data(), block->data(), SIZE));
+}