Move stuff to .cpp files

src/blockstore/implementations/encrypted/EncryptedBlockStore2.h

@@ -16,110 +16,31 @@ class EncryptedBlockStore2 final: public BlockStore2 {
 public:
   BOOST_CONCEPT_ASSERT((cpputils::CipherConcept<Cipher>));
 
-  EncryptedBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
+  EncryptedBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const typename Cipher::EncryptionKey &encKey);
-      : _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
-  }
 
-  boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override {
+  boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override;
-    cpputils::Data encrypted = _encrypt(key, data);
+  boost::future<bool> remove(const Key &key) override;
-    return _baseBlockStore->tryCreate(key, encrypted);
+  boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override;
-  }
+  boost::future<void> store(const Key &key, const cpputils::Data &data) override;
-
+  uint64_t numBlocks() const override;
-  boost::future<bool> remove(const Key &key) override {
+  uint64_t estimateNumFreeBytes() const override;
-    return _baseBlockStore->remove(key);
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
-  }
+  void forEachBlock(std::function<void (const Key &)> callback) const override;
-
-  boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override {
-    auto loaded = _baseBlockStore->load(key);
-    return loaded.then([this, key] (boost::future<boost::optional<cpputils::Data>> data_) {
-      auto data = data_.get();
-      if (boost::none == data) {
-        return boost::optional<cpputils::Data>(boost::none);
-      }
-      return _tryDecrypt(key, *data);
-    });
-  }
-
-  boost::future<void> store(const Key &key, const cpputils::Data &data) override {
-    cpputils::Data encrypted = _encrypt(key, data);
-    return _baseBlockStore->store(key, encrypted);
-  }
-
-  uint64_t numBlocks() const override {
-    return _baseBlockStore->numBlocks();
-  }
-
-  uint64_t estimateNumFreeBytes() const override {
-    return _baseBlockStore->estimateNumFreeBytes();
-  }
-
-  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override {
-    uint64_t baseBlockSize = _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
-    if (baseBlockSize <= Cipher::ciphertextSize(HEADER_LENGTH) + sizeof(FORMAT_VERSION_HEADER)) {
-      return 0;
-    }
-    return Cipher::plaintextSize(baseBlockSize - sizeof(FORMAT_VERSION_HEADER)) - HEADER_LENGTH;
-  }
-
-  void forEachBlock(std::function<void (const Key &)> callback) const override {
-    return _baseBlockStore->forEachBlock(std::move(callback));
-  }
 
 private:
 
   // This header is prepended to blocks to allow future versions to have compatibility.
   static constexpr uint16_t FORMAT_VERSION_HEADER = 0;
-
   static constexpr unsigned int HEADER_LENGTH = Key::BINARY_LENGTH;
 
-  cpputils::Data _encrypt(const Key &key, const cpputils::Data &data) const {
+  cpputils::Data _encrypt(const Key &key, const cpputils::Data &data) const;
-    cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, data);
+  boost::optional<cpputils::Data> _tryDecrypt(const Key &key, const cpputils::Data &data) const;
-    cpputils::Data encrypted = Cipher::encrypt((byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), _encKey);
-    return _prependFormatHeaderToData(encrypted);
-  }
 
-  boost::optional<cpputils::Data> _tryDecrypt(const Key &key, const cpputils::Data &data) const {
+  static cpputils::Data _prependKeyHeaderToData(const Key &key, const cpputils::Data &data);
-    auto ciphertext = _checkAndRemoveFormatHeader(data);
+  static bool _keyHeaderIsCorrect(const Key &key, const cpputils::Data &data);
-    boost::optional<cpputils::Data> decrypted = Cipher::decrypt((byte*)ciphertext.data(), ciphertext.size(), _encKey);
+  static cpputils::Data _prependFormatHeaderToData(const cpputils::Data &data);
-    if (boost::none == decrypted) {
+  static cpputils::Data _removeKeyHeader(const cpputils::Data &data);
-      // TODO Warning
+  static cpputils::Data _checkAndRemoveFormatHeader(const cpputils::Data &data);
-      return boost::none;
-    }
-    if (!_keyHeaderIsCorrect(key, *decrypted)) {
-      // TODO Warning
-      return boost::none;
-    }
-    return _removeKeyHeader(*decrypted);
-  }
-
-  static cpputils::Data _prependKeyHeaderToData(const Key &key, const cpputils::Data &data) {
-    cpputils::Data result(data.size() + Key::BINARY_LENGTH);
-    std::memcpy(result.data(), key.data(), Key::BINARY_LENGTH);
-    std::memcpy((uint8_t*)result.data() + Key::BINARY_LENGTH, data.data(), data.size());
-    return result;
-  }
-
-  static bool _keyHeaderIsCorrect(const Key &key, const cpputils::Data &data) {
-    return 0 == std::memcmp(key.data(), data.data(), Key::BINARY_LENGTH);
-  }
-
-  static cpputils::Data _prependFormatHeaderToData(const cpputils::Data &data) {
-    cpputils::Data dataWithHeader(sizeof(FORMAT_VERSION_HEADER) + data.size());
-    std::memcpy(dataWithHeader.dataOffset(0), &FORMAT_VERSION_HEADER, sizeof(FORMAT_VERSION_HEADER));
-    std::memcpy(dataWithHeader.dataOffset(sizeof(FORMAT_VERSION_HEADER)), data.data(), data.size());
-    return dataWithHeader;
-  }
-
-  static cpputils::Data _removeKeyHeader(const cpputils::Data &data) {
-    return data.copyAndRemovePrefix(Key::BINARY_LENGTH);
-  }
-
-  static cpputils::Data _checkAndRemoveFormatHeader(const cpputils::Data &data) {
-    if (*reinterpret_cast<decltype(FORMAT_VERSION_HEADER)*>(data.data()) != FORMAT_VERSION_HEADER) {
-      throw std::runtime_error("The encrypted block has the wrong format. Was it created with a newer version of CryFS?");
-    }
-    return data.copyAndRemovePrefix(sizeof(FORMAT_VERSION_HEADER));
-  }
 
   cpputils::unique_ref<BlockStore2> _baseBlockStore;
   typename Cipher::EncryptionKey _encKey;
@@ -127,6 +48,120 @@ private:
   DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore2);
 };
 
+template<class Cipher>
+inline EncryptedBlockStore2<Cipher>::EncryptedBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
+: _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
+}
+
+template<class Cipher>
+inline boost::future<bool> EncryptedBlockStore2<Cipher>::tryCreate(const Key &key, const cpputils::Data &data) {
+  cpputils::Data encrypted = _encrypt(key, data);
+  return _baseBlockStore->tryCreate(key, encrypted);
+}
+
+template<class Cipher>
+inline boost::future<bool> EncryptedBlockStore2<Cipher>::remove(const Key &key) {
+  return _baseBlockStore->remove(key);
+}
+
+template<class Cipher>
+inline boost::future<boost::optional<cpputils::Data>> EncryptedBlockStore2<Cipher>::load(const Key &key) const {
+  auto loaded = _baseBlockStore->load(key);
+  return loaded.then([this, key] (boost::future<boost::optional<cpputils::Data>> data_) {
+    auto data = data_.get();
+    if (boost::none == data) {
+      return boost::optional<cpputils::Data>(boost::none);
+    }
+    return _tryDecrypt(key, *data);
+  });
+}
+
+template<class Cipher>
+inline boost::future<void> EncryptedBlockStore2<Cipher>::store(const Key &key, const cpputils::Data &data) {
+  cpputils::Data encrypted = _encrypt(key, data);
+  return _baseBlockStore->store(key, encrypted);
+}
+
+template<class Cipher>
+inline uint64_t EncryptedBlockStore2<Cipher>::numBlocks() const {
+  return _baseBlockStore->numBlocks();
+}
+
+template<class Cipher>
+inline uint64_t EncryptedBlockStore2<Cipher>::estimateNumFreeBytes() const {
+  return _baseBlockStore->estimateNumFreeBytes();
+}
+
+template<class Cipher>
+inline uint64_t EncryptedBlockStore2<Cipher>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  uint64_t baseBlockSize = _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
+  if (baseBlockSize <= Cipher::ciphertextSize(HEADER_LENGTH) + sizeof(FORMAT_VERSION_HEADER)) {
+    return 0;
+  }
+  return Cipher::plaintextSize(baseBlockSize - sizeof(FORMAT_VERSION_HEADER)) - HEADER_LENGTH;
+}
+
+template<class Cipher>
+inline void EncryptedBlockStore2<Cipher>::forEachBlock(std::function<void (const Key &)> callback) const {
+  return _baseBlockStore->forEachBlock(std::move(callback));
+}
+
+template<class Cipher>
+inline cpputils::Data EncryptedBlockStore2<Cipher>::_encrypt(const Key &key, const cpputils::Data &data) const {
+  cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, data);
+  cpputils::Data encrypted = Cipher::encrypt((byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), _encKey);
+  return _prependFormatHeaderToData(encrypted);
+}
+
+template<class Cipher>
+inline boost::optional<cpputils::Data> EncryptedBlockStore2<Cipher>::_tryDecrypt(const Key &key, const cpputils::Data &data) const {
+  auto ciphertext = _checkAndRemoveFormatHeader(data);
+  boost::optional<cpputils::Data> decrypted = Cipher::decrypt((byte*)ciphertext.data(), ciphertext.size(), _encKey);
+  if (boost::none == decrypted) {
+    // TODO Warning
+    return boost::none;
+  }
+  if (!_keyHeaderIsCorrect(key, *decrypted)) {
+    // TODO Warning
+    return boost::none;
+  }
+  return _removeKeyHeader(*decrypted);
+}
+
+template<class Cipher>
+inline cpputils::Data EncryptedBlockStore2<Cipher>::_prependKeyHeaderToData(const Key &key, const cpputils::Data &data) {
+  cpputils::Data result(data.size() + Key::BINARY_LENGTH);
+  std::memcpy(result.data(), key.data(), Key::BINARY_LENGTH);
+  std::memcpy((uint8_t*)result.data() + Key::BINARY_LENGTH, data.data(), data.size());
+  return result;
+}
+
+template<class Cipher>
+inline bool EncryptedBlockStore2<Cipher>::_keyHeaderIsCorrect(const Key &key, const cpputils::Data &data) {
+  return 0 == std::memcmp(key.data(), data.data(), Key::BINARY_LENGTH);
+}
+
+template<class Cipher>
+inline cpputils::Data EncryptedBlockStore2<Cipher>::_prependFormatHeaderToData(const cpputils::Data &data) {
+  cpputils::Data dataWithHeader(sizeof(FORMAT_VERSION_HEADER) + data.size());
+  std::memcpy(dataWithHeader.dataOffset(0), &FORMAT_VERSION_HEADER, sizeof(FORMAT_VERSION_HEADER));
+  std::memcpy(dataWithHeader.dataOffset(sizeof(FORMAT_VERSION_HEADER)), data.data(), data.size());
+  return dataWithHeader;
+}
+
+template<class Cipher>
+inline cpputils::Data EncryptedBlockStore2<Cipher>::_removeKeyHeader(const cpputils::Data &data) {
+  return data.copyAndRemovePrefix(Key::BINARY_LENGTH);
+}
+
+template<class Cipher>
+inline cpputils::Data EncryptedBlockStore2<Cipher>::_checkAndRemoveFormatHeader(const cpputils::Data &data) {
+  if (*reinterpret_cast<decltype(FORMAT_VERSION_HEADER)*>(data.data()) != FORMAT_VERSION_HEADER) {
+    throw std::runtime_error("The encrypted block has the wrong format. Was it created with a newer version of CryFS?");
+  }
+  return data.copyAndRemovePrefix(sizeof(FORMAT_VERSION_HEADER));
+}
+
 }
 }
 

src/blockstore/implementations/low2highlevel/LowToHighLevelBlock.cpp

@@ -3,5 +3,73 @@
 namespace blockstore {
 namespace lowtohighlevel {
 
+boost::optional<cpputils::unique_ref<LowToHighLevelBlock>> LowToHighLevelBlock::TryCreateNew(BlockStore2 *baseBlockStore, const Key &key, cpputils::Data data) {
+  // TODO .get() is blocking
+  bool success = baseBlockStore->tryCreate(key, data.copy()).get(); // TODO Copy necessary?
+  if (!success) {
+    return boost::none;
+  }
+
+  return cpputils::make_unique_ref<LowToHighLevelBlock>(key, std::move(data), baseBlockStore);
+}
+
+cpputils::unique_ref<LowToHighLevelBlock> LowToHighLevelBlock::Overwrite(BlockStore2 *baseBlockStore, const Key &key, cpputils::Data data) {
+  auto baseBlock = baseBlockStore->store(key, data); // TODO Does it make sense to not store here, but only write back in the destructor of LowToHighLevelBlock? Also: What about tryCreate?
+  return cpputils::make_unique_ref<LowToHighLevelBlock>(key, std::move(data), baseBlockStore);
+}
+
+boost::optional<cpputils::unique_ref<LowToHighLevelBlock>> LowToHighLevelBlock::Load(BlockStore2 *baseBlockStore, const Key &key) {
+  boost::optional<cpputils::Data> loadedData = baseBlockStore->load(key).get(); // TODO .get() is blocking
+  if (loadedData == boost::none) {
+    return boost::none;
+  }
+  return cpputils::make_unique_ref<LowToHighLevelBlock>(key, std::move(*loadedData), baseBlockStore);
+}
+
+LowToHighLevelBlock::LowToHighLevelBlock(const Key& key, cpputils::Data data, BlockStore2 *baseBlockStore)
+    :Block(key),
+     _baseBlockStore(baseBlockStore),
+     _data(std::move(data)),
+     _dataChanged(false),
+     _mutex() {
+}
+
+LowToHighLevelBlock::~LowToHighLevelBlock() {
+  std::unique_lock<std::mutex> lock(_mutex);
+  _storeToBaseBlock();
+}
+
+const void *LowToHighLevelBlock::data() const {
+  return (uint8_t*)_data.data();
+}
+
+void LowToHighLevelBlock::write(const void *source, uint64_t offset, uint64_t count) {
+  ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition
+  std::memcpy((uint8_t*)_data.data()+offset, source, count);
+  _dataChanged = true;
+}
+
+void LowToHighLevelBlock::flush() {
+  std::unique_lock<std::mutex> lock(_mutex);
+  _storeToBaseBlock();
+}
+
+size_t LowToHighLevelBlock::size() const {
+  return _data.size();
+}
+
+void LowToHighLevelBlock::resize(size_t newSize) {
+  _data = cpputils::DataUtils::resize(std::move(_data), newSize);
+  _dataChanged = true;
+}
+
+void LowToHighLevelBlock::_storeToBaseBlock() {
+  if (_dataChanged) {
+    _baseBlockStore->store(key(), _data);
+    _dataChanged = false;
+  }
+}
+
+
 }
 }

src/blockstore/implementations/low2highlevel/LowToHighLevelBlock.h

@@ -48,74 +48,6 @@ private:
   DISALLOW_COPY_AND_ASSIGN(LowToHighLevelBlock);
 };
 
-
-inline boost::optional<cpputils::unique_ref<LowToHighLevelBlock>> LowToHighLevelBlock::TryCreateNew(BlockStore2 *baseBlockStore, const Key &key, cpputils::Data data) {
-  // TODO .get() is blocking
-  bool success = baseBlockStore->tryCreate(key, data.copy()).get(); // TODO Copy necessary?
-  if (!success) {
-    return boost::none;
-  }
-
-  return cpputils::make_unique_ref<LowToHighLevelBlock>(key, std::move(data), baseBlockStore);
-}
-
-inline cpputils::unique_ref<LowToHighLevelBlock> LowToHighLevelBlock::Overwrite(BlockStore2 *baseBlockStore, const Key &key, cpputils::Data data) {
-  auto baseBlock = baseBlockStore->store(key, data); // TODO Does it make sense to not store here, but only write back in the destructor of LowToHighLevelBlock? Also: What about tryCreate?
-  return cpputils::make_unique_ref<LowToHighLevelBlock>(key, std::move(data), baseBlockStore);
-}
-
-inline boost::optional<cpputils::unique_ref<LowToHighLevelBlock>> LowToHighLevelBlock::Load(BlockStore2 *baseBlockStore, const Key &key) {
-  boost::optional<cpputils::Data> loadedData = baseBlockStore->load(key).get(); // TODO .get() is blocking
-  if (loadedData == boost::none) {
-    return boost::none;
-  }
-  return cpputils::make_unique_ref<LowToHighLevelBlock>(key, std::move(*loadedData), baseBlockStore);
-}
-
-inline LowToHighLevelBlock::LowToHighLevelBlock(const Key& key, cpputils::Data data, BlockStore2 *baseBlockStore)
-    :Block(key),
-   _baseBlockStore(baseBlockStore),
-   _data(std::move(data)),
-   _dataChanged(false),
-   _mutex() {
-}
-
-inline LowToHighLevelBlock::~LowToHighLevelBlock() {
-  std::unique_lock<std::mutex> lock(_mutex);
-  _storeToBaseBlock();
-}
-
-inline const void *LowToHighLevelBlock::data() const {
-  return (uint8_t*)_data.data();
-}
-
-inline void LowToHighLevelBlock::write(const void *source, uint64_t offset, uint64_t count) {
-  ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition
-  std::memcpy((uint8_t*)_data.data()+offset, source, count);
-  _dataChanged = true;
-}
-
-inline void LowToHighLevelBlock::flush() {
-  std::unique_lock<std::mutex> lock(_mutex);
-  _storeToBaseBlock();
-}
-
-inline size_t LowToHighLevelBlock::size() const {
-  return _data.size();
-}
-
-inline void LowToHighLevelBlock::resize(size_t newSize) {
-  _data = cpputils::DataUtils::resize(std::move(_data), newSize);
-  _dataChanged = true;
-}
-
-inline void LowToHighLevelBlock::_storeToBaseBlock() {
-  if (_dataChanged) {
-    _baseBlockStore->store(key(), _data);
-    _dataChanged = false;
-  }
-}
-
 }
 }
 

src/blockstore/implementations/ondisk/OnDiskBlockStore2.cpp

@@ -8,5 +8,117 @@ namespace ondisk {
 const string OnDiskBlockStore2::FORMAT_VERSION_HEADER_PREFIX = "cryfs;block;";
 const string OnDiskBlockStore2::FORMAT_VERSION_HEADER = OnDiskBlockStore2::FORMAT_VERSION_HEADER_PREFIX + "0";
 
+boost::filesystem::path OnDiskBlockStore2::_getFilepath(const Key &key) const {
+  std::string keyStr = key.ToString();
+  return _rootDir / keyStr.substr(0,3) / keyStr.substr(3);
+}
+
+cpputils::Data OnDiskBlockStore2::_checkAndRemoveHeader(const cpputils::Data &data) {
+  if (!_isAcceptedCryfsHeader(data)) {
+    if (_isOtherCryfsHeader(data)) {
+      throw std::runtime_error("This block is not supported yet. Maybe it was created with a newer version of CryFS?");
+    } else {
+      throw std::runtime_error("This is not a valid block.");
+    }
+  }
+  cpputils::Data result(data.size() - formatVersionHeaderSize());
+  std::memcpy(result.data(), data.dataOffset(formatVersionHeaderSize()), result.size());
+  return result;
+}
+
+bool OnDiskBlockStore2::_isAcceptedCryfsHeader(const cpputils::Data &data) {
+  return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
+}
+
+bool OnDiskBlockStore2::_isOtherCryfsHeader(const cpputils::Data &data) {
+  return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER_PREFIX.c_str(), FORMAT_VERSION_HEADER_PREFIX.size());
+}
+
+unsigned int OnDiskBlockStore2::formatVersionHeaderSize() {
+  return FORMAT_VERSION_HEADER.size() + 1; // +1 because of the null byte
+}
+
+OnDiskBlockStore2::OnDiskBlockStore2(const boost::filesystem::path& path)
+    : _rootDir(path) {}
+
+boost::future<bool> OnDiskBlockStore2::tryCreate(const Key &key, const cpputils::Data &data) {
+  auto filepath = _getFilepath(key);
+  if (boost::filesystem::exists(filepath)) {
+    return boost::make_ready_future(false);
+  }
+
+  store(key, data).wait();
+  return boost::make_ready_future(true);
+}
+
+boost::future<bool> OnDiskBlockStore2::remove(const Key &key) {
+  auto filepath = _getFilepath(key);
+  if (!boost::filesystem::is_regular_file(filepath)) { // TODO Is this branch necessary?
+    return boost::make_ready_future(false);
+  }
+  bool retval = boost::filesystem::remove(filepath);
+  if (!retval) {
+    cpputils::logging::LOG(cpputils::logging::ERROR, "Couldn't find block {} to remove", key.ToString());
+    return boost::make_ready_future(false);
+  }
+  if (boost::filesystem::is_empty(filepath.parent_path())) {
+    boost::filesystem::remove(filepath.parent_path());
+  }
+  return boost::make_ready_future(true);
+}
+
+boost::future<boost::optional<cpputils::Data>> OnDiskBlockStore2::load(const Key &key) const {
+  auto fileContent = cpputils::Data::LoadFromFile(_getFilepath(key));
+  if (fileContent == boost::none) {
+    return boost::make_ready_future(boost::optional<cpputils::Data>(boost::none));
+  }
+  return boost::make_ready_future(boost::optional<cpputils::Data>(_checkAndRemoveHeader(std::move(*fileContent))));
+}
+
+boost::future<void> OnDiskBlockStore2::store(const Key &key, const cpputils::Data &data) {
+  cpputils::Data fileContent(formatVersionHeaderSize() + data.size());
+  std::memcpy(fileContent.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
+  std::memcpy(fileContent.dataOffset(formatVersionHeaderSize()), data.data(), data.size());
+  auto filepath = _getFilepath(key);
+  boost::filesystem::create_directory(filepath.parent_path()); // TODO Instead create all of them once at fs creation time?
+  fileContent.StoreToFile(filepath);
+  return boost::make_ready_future();
+}
+
+uint64_t OnDiskBlockStore2::numBlocks() const {
+  uint64_t count = 0;
+  for (auto prefixDir = boost::filesystem::directory_iterator(_rootDir); prefixDir != boost::filesystem::directory_iterator(); ++prefixDir) {
+    if (boost::filesystem::is_directory(prefixDir->path())) {
+      count += std::distance(boost::filesystem::directory_iterator(prefixDir->path()), boost::filesystem::directory_iterator());
+    }
+  }
+  return count;
+}
+
+uint64_t OnDiskBlockStore2::estimateNumFreeBytes() const {
+  struct statvfs stat;
+  ::statvfs(_rootDir.c_str(), &stat);
+  return stat.f_bsize*stat.f_bavail;
+}
+
+uint64_t OnDiskBlockStore2::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  if(blockSize <= formatVersionHeaderSize()) {
+    return 0;
+  }
+  return blockSize - formatVersionHeaderSize();
+}
+
+void OnDiskBlockStore2::forEachBlock(std::function<void (const Key &)> callback) const {
+  for (auto prefixDir = boost::filesystem::directory_iterator(_rootDir); prefixDir != boost::filesystem::directory_iterator(); ++prefixDir) {
+    if (boost::filesystem::is_directory(prefixDir->path())) {
+      std::string blockKeyPrefix = prefixDir->path().filename().native();
+      for (auto block = boost::filesystem::directory_iterator(prefixDir->path()); block != boost::filesystem::directory_iterator(); ++block) {
+        std::string blockKeyPostfix = block->path().filename().native();
+        callback(Key::FromString(blockKeyPrefix + blockKeyPostfix));
+      }
+    }
+  }
+}
+
 }
 }

src/blockstore/implementations/ondisk/OnDiskBlockStore2.h

@@ -15,87 +15,16 @@ namespace ondisk {
 
 class OnDiskBlockStore2 final: public BlockStore2 {
 public:
-  explicit OnDiskBlockStore2(const boost::filesystem::path& path)
+  explicit OnDiskBlockStore2(const boost::filesystem::path& path);
-    : _rootDir(path) {}
 
-  boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override {
+  boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override;
-    auto filepath = _getFilepath(key);
+  boost::future<bool> remove(const Key &key) override;
-    if (boost::filesystem::exists(filepath)) {
+  boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override;
-      return boost::make_ready_future(false);
+  boost::future<void> store(const Key &key, const cpputils::Data &data) override;
-    }
+  uint64_t numBlocks() const override;
-
+  uint64_t estimateNumFreeBytes() const override;
-    store(key, data).wait();
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
-    return boost::make_ready_future(true);
+  void forEachBlock(std::function<void (const Key &)> callback) const override;
-  }
-
-  boost::future<bool> remove(const Key &key) override {
-    auto filepath = _getFilepath(key);
-    if (!boost::filesystem::is_regular_file(filepath)) { // TODO Is this branch necessary?
-      return boost::make_ready_future(false);
-    }
-    bool retval = boost::filesystem::remove(filepath);
-    if (!retval) {
-      cpputils::logging::LOG(cpputils::logging::ERROR, "Couldn't find block {} to remove", key.ToString());
-      return boost::make_ready_future(false);
-    }
-    if (boost::filesystem::is_empty(filepath.parent_path())) {
-      boost::filesystem::remove(filepath.parent_path());
-    }
-    return boost::make_ready_future(true);
-  }
-
-  boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override {
-    auto fileContent = cpputils::Data::LoadFromFile(_getFilepath(key));
-    if (fileContent == boost::none) {
-      return boost::make_ready_future(boost::optional<cpputils::Data>(boost::none));
-    }
-    return boost::make_ready_future(boost::optional<cpputils::Data>(_checkAndRemoveHeader(std::move(*fileContent))));
-  }
-
-  boost::future<void> store(const Key &key, const cpputils::Data &data) override {
-    cpputils::Data fileContent(formatVersionHeaderSize() + data.size());
-    std::memcpy(fileContent.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
-    std::memcpy(fileContent.dataOffset(formatVersionHeaderSize()), data.data(), data.size());
-    auto filepath = _getFilepath(key);
-    boost::filesystem::create_directory(filepath.parent_path()); // TODO Instead create all of them once at fs creation time?
-    fileContent.StoreToFile(filepath);
-    return boost::make_ready_future();
-  }
-
-  uint64_t numBlocks() const override {
-    uint64_t count = 0;
-    for (auto prefixDir = boost::filesystem::directory_iterator(_rootDir); prefixDir != boost::filesystem::directory_iterator(); ++prefixDir) {
-      if (boost::filesystem::is_directory(prefixDir->path())) {
-        count += std::distance(boost::filesystem::directory_iterator(prefixDir->path()), boost::filesystem::directory_iterator());
-      }
-    }
-    return count;
-  }
-
-  uint64_t estimateNumFreeBytes() const override {
-    struct statvfs stat;
-    ::statvfs(_rootDir.c_str(), &stat);
-    return stat.f_bsize*stat.f_bavail;
-  }
-
-  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override {
-    if(blockSize <= formatVersionHeaderSize()) {
-      return 0;
-    }
-    return blockSize - formatVersionHeaderSize();
-  }
-
-  void forEachBlock(std::function<void (const Key &)> callback) const override {
-    for (auto prefixDir = boost::filesystem::directory_iterator(_rootDir); prefixDir != boost::filesystem::directory_iterator(); ++prefixDir) {
-      if (boost::filesystem::is_directory(prefixDir->path())) {
-        std::string blockKeyPrefix = prefixDir->path().filename().native();
-        for (auto block = boost::filesystem::directory_iterator(prefixDir->path()); block != boost::filesystem::directory_iterator(); ++block) {
-          std::string blockKeyPostfix = block->path().filename().native();
-          callback(Key::FromString(blockKeyPrefix + blockKeyPostfix));
-        }
-      }
-    }
-  }
 
 private:
   boost::filesystem::path _rootDir;
@@ -103,35 +32,11 @@ private:
   static const std::string FORMAT_VERSION_HEADER_PREFIX;
   static const std::string FORMAT_VERSION_HEADER;
 
-  boost::filesystem::path _getFilepath(const Key &key) const {
+  boost::filesystem::path _getFilepath(const Key &key) const;
-    std::string keyStr = key.ToString();
+  static cpputils::Data _checkAndRemoveHeader(const cpputils::Data &data);
-    return _rootDir / keyStr.substr(0,3) / keyStr.substr(3);
+  static bool _isAcceptedCryfsHeader(const cpputils::Data &data);
-  }
+  static bool _isOtherCryfsHeader(const cpputils::Data &data);
-
+  static unsigned int formatVersionHeaderSize();
-  static cpputils::Data _checkAndRemoveHeader(const cpputils::Data &data) {
-    if (!_isAcceptedCryfsHeader(data)) {
-      if (_isOtherCryfsHeader(data)) {
-        throw std::runtime_error("This block is not supported yet. Maybe it was created with a newer version of CryFS?");
-      } else {
-        throw std::runtime_error("This is not a valid block.");
-      }
-    }
-    cpputils::Data result(data.size() - formatVersionHeaderSize());
-    std::memcpy(result.data(), data.dataOffset(formatVersionHeaderSize()), result.size());
-    return result;
-  }
-
-  static bool _isAcceptedCryfsHeader(const cpputils::Data &data) {
-    return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
-  }
-
-  static bool _isOtherCryfsHeader(const cpputils::Data &data) {
-    return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER_PREFIX.c_str(), FORMAT_VERSION_HEADER_PREFIX.size());
-  }
-
-  static unsigned int formatVersionHeaderSize() {
-    return FORMAT_VERSION_HEADER.size() + 1; // +1 because of the null byte
-  }
 
   DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore2);
 };

src/blockstore/implementations/versioncounting/VersionCountingBlockStore2.cpp

@@ -1 +1,142 @@
 #include "VersionCountingBlockStore2.h"
+
+namespace blockstore {
+namespace versioncounting {
+
+cpputils::Data VersionCountingBlockStore2::_prependHeaderToData(uint32_t myClientId, uint64_t version, const cpputils::Data &data) {
+  static_assert(HEADER_LENGTH == sizeof(FORMAT_VERSION_HEADER) + sizeof(myClientId) + sizeof(version), "Wrong header length");
+  cpputils::Data result(data.size() + HEADER_LENGTH);
+  std::memcpy(result.dataOffset(0), &FORMAT_VERSION_HEADER, sizeof(FORMAT_VERSION_HEADER));
+  std::memcpy(result.dataOffset(CLIENTID_HEADER_OFFSET), &myClientId, sizeof(myClientId));
+  std::memcpy(result.dataOffset(VERSION_HEADER_OFFSET), &version, sizeof(version));
+  std::memcpy((uint8_t*)result.dataOffset(HEADER_LENGTH), data.data(), data.size());
+  return result;
+}
+
+void VersionCountingBlockStore2::_checkHeader(const Key &key, const cpputils::Data &data) const {
+  _checkFormatHeader(data);
+  _checkVersionHeader(key, data);
+}
+
+void VersionCountingBlockStore2::_checkFormatHeader(const cpputils::Data &data) const {
+  if (*reinterpret_cast<decltype(FORMAT_VERSION_HEADER)*>(data.data()) != FORMAT_VERSION_HEADER) {
+    throw std::runtime_error("The versioned block has the wrong format. Was it created with a newer version of CryFS?");
+  }
+}
+
+void VersionCountingBlockStore2::_checkVersionHeader(const Key &key, const cpputils::Data &data) const {
+  uint32_t clientId = _readClientId(data);
+  uint64_t version = _readVersion(data);
+
+  if(!_knownBlockVersions.checkAndUpdateVersion(clientId, key, version)) {
+    integrityViolationDetected("The block version number is too low. Did an attacker try to roll back the block or to re-introduce a deleted block?");
+  }
+}
+
+uint32_t VersionCountingBlockStore2::_readClientId(const cpputils::Data &data) {
+  uint32_t clientId;
+  std::memcpy(&clientId, data.dataOffset(CLIENTID_HEADER_OFFSET), sizeof(clientId));
+  return clientId;
+}
+
+uint64_t VersionCountingBlockStore2::_readVersion(const cpputils::Data &data) {
+  uint64_t version;
+  std::memcpy(&version, data.dataOffset(VERSION_HEADER_OFFSET), sizeof(version));
+  return version;
+}
+
+cpputils::Data VersionCountingBlockStore2::_removeHeader(const cpputils::Data &data) const {
+  return data.copyAndRemovePrefix(HEADER_LENGTH);
+}
+
+void VersionCountingBlockStore2::_checkNoPastIntegrityViolations() const {
+  if (_integrityViolationDetected) {
+    throw std::runtime_error(std::string() +
+                             "There was an integrity violation detected. Preventing any further access to the file system. " +
+                             "If you want to reset the integrity data (i.e. accept changes made by a potential attacker), " +
+                             "please unmount the file system and delete the following file before re-mounting it: " +
+                             _knownBlockVersions.path().native());
+  }
+}
+
+void VersionCountingBlockStore2::integrityViolationDetected(const std::string &reason) const {
+  _integrityViolationDetected = true;
+  throw IntegrityViolationError(reason);
+}
+
+VersionCountingBlockStore2::VersionCountingBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId, bool missingBlockIsIntegrityViolation)
+: _baseBlockStore(std::move(baseBlockStore)), _knownBlockVersions(integrityFilePath, myClientId), _missingBlockIsIntegrityViolation(missingBlockIsIntegrityViolation), _integrityViolationDetected(false) {
+}
+
+boost::future<bool> VersionCountingBlockStore2::tryCreate(const Key &key, const cpputils::Data &data) {
+  _checkNoPastIntegrityViolations();
+  uint64_t version = _knownBlockVersions.incrementVersion(key);
+  cpputils::Data dataWithHeader = _prependHeaderToData(_knownBlockVersions.myClientId(), version, data);
+  return _baseBlockStore->tryCreate(key, dataWithHeader);
+}
+
+boost::future<bool> VersionCountingBlockStore2::remove(const Key &key) {
+  _checkNoPastIntegrityViolations();
+  _knownBlockVersions.markBlockAsDeleted(key);
+  return _baseBlockStore->remove(key);
+}
+
+boost::future<boost::optional<cpputils::Data>> VersionCountingBlockStore2::load(const Key &key) const {
+  _checkNoPastIntegrityViolations();
+  return _baseBlockStore->load(key).then([this, key] (boost::future<boost::optional<cpputils::Data>> loaded_) {
+    auto loaded = loaded_.get();
+    if (boost::none == loaded) {
+      if (_missingBlockIsIntegrityViolation && _knownBlockVersions.blockShouldExist(key)) {
+        integrityViolationDetected("A block that should exist wasn't found. Did an attacker delete it?");
+      }
+      return boost::optional<cpputils::Data>(boost::none);
+    }
+    _checkHeader(key, *loaded);
+    return boost::optional<cpputils::Data>(_removeHeader(*loaded));
+  });
+}
+
+boost::future<void> VersionCountingBlockStore2::store(const Key &key, const cpputils::Data &data) {
+  _checkNoPastIntegrityViolations();
+  uint64_t version = _knownBlockVersions.incrementVersion(key);
+  cpputils::Data dataWithHeader = _prependHeaderToData(_knownBlockVersions.myClientId(), version, data);
+  return _baseBlockStore->store(key, dataWithHeader);
+}
+
+uint64_t VersionCountingBlockStore2::numBlocks() const {
+  return _baseBlockStore->numBlocks();
+}
+
+uint64_t VersionCountingBlockStore2::estimateNumFreeBytes() const {
+  return _baseBlockStore->estimateNumFreeBytes();
+}
+
+uint64_t VersionCountingBlockStore2::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  uint64_t baseBlockSize = _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
+  if (baseBlockSize <= HEADER_LENGTH) {
+    return 0;
+  }
+  return baseBlockSize - HEADER_LENGTH;
+}
+
+void VersionCountingBlockStore2::forEachBlock(std::function<void (const Key &)> callback) const {
+  if (!_missingBlockIsIntegrityViolation) {
+    return _baseBlockStore->forEachBlock(std::move(callback));
+  }
+
+  std::unordered_set<blockstore::Key> existingBlocks = _knownBlockVersions.existingBlocks();
+  _baseBlockStore->forEachBlock([&existingBlocks, callback] (const Key &key) {
+    callback(key);
+
+    auto found = existingBlocks.find(key);
+    if (found != existingBlocks.end()) {
+      existingBlocks.erase(found);
+    }
+  });
+  if (!existingBlocks.empty()) {
+    integrityViolationDetected("A block that should have existed wasn't found.");
+  }
+}
+
+}
+}

src/blockstore/implementations/versioncounting/VersionCountingBlockStore2.h

@@ -14,79 +14,16 @@ namespace versioncounting {
 
 class VersionCountingBlockStore2 final: public BlockStore2 {
 public:
-  VersionCountingBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId, bool missingBlockIsIntegrityViolation)
+  VersionCountingBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId, bool missingBlockIsIntegrityViolation);
-      : _baseBlockStore(std::move(baseBlockStore)), _knownBlockVersions(integrityFilePath, myClientId), _missingBlockIsIntegrityViolation(missingBlockIsIntegrityViolation), _integrityViolationDetected(false) {
-  }
 
-  boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override {
+  boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override;
-    _checkNoPastIntegrityViolations();
+  boost::future<bool> remove(const Key &key) override;
-    uint64_t version = _knownBlockVersions.incrementVersion(key);
+  boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override;
-    cpputils::Data dataWithHeader = _prependHeaderToData(_knownBlockVersions.myClientId(), version, data);
+  boost::future<void> store(const Key &key, const cpputils::Data &data) override;
-    return _baseBlockStore->tryCreate(key, dataWithHeader);
+  uint64_t numBlocks() const override;
-  }
+  uint64_t estimateNumFreeBytes() const override;
-
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
-  boost::future<bool> remove(const Key &key) override {
+  void forEachBlock(std::function<void (const Key &)> callback) const override;
-    _checkNoPastIntegrityViolations();
-    _knownBlockVersions.markBlockAsDeleted(key);
-    return _baseBlockStore->remove(key);
-  }
-
-  boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override {
-    _checkNoPastIntegrityViolations();
-    return _baseBlockStore->load(key).then([this, key] (boost::future<boost::optional<cpputils::Data>> loaded_) {
-      auto loaded = loaded_.get();
-      if (boost::none == loaded) {
-        if (_missingBlockIsIntegrityViolation && _knownBlockVersions.blockShouldExist(key)) {
-          integrityViolationDetected("A block that should exist wasn't found. Did an attacker delete it?");
-        }
-        return boost::optional<cpputils::Data>(boost::none);
-      }
-      _checkHeader(key, *loaded);
-      return boost::optional<cpputils::Data>(_removeHeader(*loaded));
-    });
-  }
-
-  boost::future<void> store(const Key &key, const cpputils::Data &data) override {
-    _checkNoPastIntegrityViolations();
-    uint64_t version = _knownBlockVersions.incrementVersion(key);
-    cpputils::Data dataWithHeader = _prependHeaderToData(_knownBlockVersions.myClientId(), version, data);
-    return _baseBlockStore->store(key, dataWithHeader);
-  }
-
-  uint64_t numBlocks() const override {
-    return _baseBlockStore->numBlocks();
-  }
-
-  uint64_t estimateNumFreeBytes() const override {
-    return _baseBlockStore->estimateNumFreeBytes();
-  }
-
-  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override {
-    uint64_t baseBlockSize = _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
-    if (baseBlockSize <= HEADER_LENGTH) {
-      return 0;
-    }
-    return baseBlockSize - HEADER_LENGTH;
-  }
-
-  void forEachBlock(std::function<void (const Key &)> callback) const override {
-    if (!_missingBlockIsIntegrityViolation) {
-      return _baseBlockStore->forEachBlock(std::move(callback));
-    }
-
-    std::unordered_set<blockstore::Key> existingBlocks = _knownBlockVersions.existingBlocks();
-    _baseBlockStore->forEachBlock([&existingBlocks, callback] (const Key &key) {
-      callback(key);
-
-      auto found = existingBlocks.find(key);
-      if (found != existingBlocks.end()) {
-        existingBlocks.erase(found);
-      }
-    });
-    if (!existingBlocks.empty()) {
-      integrityViolationDetected("A block that should have existed wasn't found.");
-    }
-  }
 
 private:
   // This header is prepended to blocks to allow future versions to have compatibility.
@@ -100,66 +37,15 @@ public:
 
 private:
 
-  cpputils::Data _prependHeaderToData(uint32_t myClientId, uint64_t version, const cpputils::Data &data) {
+  cpputils::Data _prependHeaderToData(uint32_t myClientId, uint64_t version, const cpputils::Data &data);
-    static_assert(HEADER_LENGTH == sizeof(FORMAT_VERSION_HEADER) + sizeof(myClientId) + sizeof(version), "Wrong header length");
+  void _checkHeader(const Key &key, const cpputils::Data &data) const;
-    cpputils::Data result(data.size() + HEADER_LENGTH);
+  void _checkFormatHeader(const cpputils::Data &data) const;
-    std::memcpy(result.dataOffset(0), &FORMAT_VERSION_HEADER, sizeof(FORMAT_VERSION_HEADER));
+  void _checkVersionHeader(const Key &key, const cpputils::Data &data) const;
-    std::memcpy(result.dataOffset(CLIENTID_HEADER_OFFSET), &myClientId, sizeof(myClientId));
+  static uint32_t _readClientId(const cpputils::Data &data);
-    std::memcpy(result.dataOffset(VERSION_HEADER_OFFSET), &version, sizeof(version));
+  static uint64_t _readVersion(const cpputils::Data &data);
-    std::memcpy((uint8_t*)result.dataOffset(HEADER_LENGTH), data.data(), data.size());
+  cpputils::Data _removeHeader(const cpputils::Data &data) const;
-    return result;
+  void _checkNoPastIntegrityViolations() const;
-  }
+  void integrityViolationDetected(const std::string &reason) const;
-
-  void _checkHeader(const Key &key, const cpputils::Data &data) const {
-    _checkFormatHeader(data);
-    _checkVersionHeader(key, data);
-  }
-
-  void _checkFormatHeader(const cpputils::Data &data) const {
-    if (*reinterpret_cast<decltype(FORMAT_VERSION_HEADER)*>(data.data()) != FORMAT_VERSION_HEADER) {
-      throw std::runtime_error("The versioned block has the wrong format. Was it created with a newer version of CryFS?");
-    }
-  }
-
-  void _checkVersionHeader(const Key &key, const cpputils::Data &data) const {
-    uint32_t clientId = _readClientId(data);
-    uint64_t version = _readVersion(data);
-
-    if(!_knownBlockVersions.checkAndUpdateVersion(clientId, key, version)) {
-      integrityViolationDetected("The block version number is too low. Did an attacker try to roll back the block or to re-introduce a deleted block?");
-    }
-  }
-
-  static uint32_t _readClientId(const cpputils::Data &data) {
-    uint32_t clientId;
-    std::memcpy(&clientId, data.dataOffset(CLIENTID_HEADER_OFFSET), sizeof(clientId));
-    return clientId;
-  }
-
-  static uint64_t _readVersion(const cpputils::Data &data) {
-    uint64_t version;
-    std::memcpy(&version, data.dataOffset(VERSION_HEADER_OFFSET), sizeof(version));
-    return version;
-  }
-
-  cpputils::Data _removeHeader(const cpputils::Data &data) const {
-    return data.copyAndRemovePrefix(HEADER_LENGTH);
-  }
-
-  void _checkNoPastIntegrityViolations() const {
-    if (_integrityViolationDetected) {
-      throw std::runtime_error(std::string() +
-                               "There was an integrity violation detected. Preventing any further access to the file system. " +
-                               "If you want to reset the integrity data (i.e. accept changes made by a potential attacker), " +
-                               "please unmount the file system and delete the following file before re-mounting it: " +
-                               _knownBlockVersions.path().native());
-    }
-  }
-
-  void integrityViolationDetected(const std::string &reason) const {
-    _integrityViolationDetected = true;
-    throw IntegrityViolationError(reason);
-  }
 
   cpputils::unique_ref<BlockStore2> _baseBlockStore;
   mutable KnownBlockVersions _knownBlockVersions;