Start implementing new blockstore

src/blockstore/CMakeLists.txt

@@ -17,8 +17,10 @@ set(SOURCES
   implementations/compressing/compressors/RunLengthEncoding.cpp
   implementations/compressing/compressors/Gzip.cpp
   implementations/encrypted/EncryptedBlockStore.cpp
+  implementations/encrypted/EncryptedBlockStore2.cpp
   implementations/encrypted/EncryptedBlock.cpp
   implementations/ondisk/OnDiskBlockStore.cpp
+  implementations/ondisk/OnDiskBlockStore2.cpp
   implementations/ondisk/OnDiskBlock.cpp
   implementations/caching/CachingBlockStore.cpp
   implementations/caching/cache/PeriodicTask.cpp

src/blockstore/implementations/encrypted/EncryptedBlockStore2.cpp

@@ -0,0 +1 @@
+#include "EncryptedBlockStore2.h"

src/blockstore/implementations/encrypted/EncryptedBlockStore2.h

@@ -0,0 +1,84 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE2_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE2_H_
+
+#include "../../interface/BlockStore2.h"
+#include <cpp-utils/macros.h>
+#include <cpp-utils/crypto/symmetric/Cipher.h>
+
+namespace blockstore {
+namespace encrypted {
+
+//TODO Format version headers
+
+template<class Cipher>
+class EncryptedBlockStore2 final: public BlockStore2 {
+public:
+  BOOST_CONCEPT_ASSERT((cpputils::CipherConcept<Cipher>));
+
+  EncryptedBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
+      : _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
+  }
+
+  boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override {
+    cpputils::Data encrypted = _encrypt(data);
+    return _baseBlockStore->tryCreate(key, encrypted);
+  }
+
+  boost::future<bool> remove(const Key &key) override {
+    return _baseBlockStore->remove(key);
+  }
+
+  boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override {
+    auto loaded = _baseBlockStore->load(key);
+    return loaded.then([this] (boost::future<boost::optional<cpputils::Data>> data_) {
+      auto data = data_.get();
+      if (boost::none == data) {
+        return boost::optional<cpputils::Data>(boost::none);
+      }
+      return _tryDecrypt(data);
+    });
+  }
+
+  boost::future<void> store(const Key &key, const cpputils::Data &data) override {
+    cpputils::Data encrypted = _encrypt(data);
+    return _baseBlockStore->store(key, encrypted);
+  }
+
+private:
+
+  cpputils::Data _encrypt(const Key &key, const cpputils::Data &data) {
+    cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, data);
+    return Cipher::encrypt((byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), _encKey);
+  }
+
+  boost::optional<cpputils::Data> _tryDecrypt(const Key &key, const cpputils::Data &data) {
+    boost::optional<cpputils::Data> decrypted = Cipher::decrypt((byte*)data.data(), data.size(), _encKey);
+    if (boost::none != decrypted && !_keyHeaderIsCorrect(key, *decrypted)) {
+      // TODO Warning
+      return boost::none;
+    }
+    return decrypted;
+  }
+
+  static cpputils::Data _prependKeyHeaderToData(const Key &key, const cpputils::Data &data) {
+    cpputils::Data result(data.size() + Key::BINARY_LENGTH);
+    std::memcpy(result.data(), key.data(), Key::BINARY_LENGTH);
+    std::memcpy((uint8_t*)result.data() + Key::BINARY_LENGTH, data.data(), data.size());
+    return result;
+  }
+
+  static bool _keyHeaderIsCorrect(const Key &key, const cpputils::Data &data) {
+    return 0 == std::memcmp(key.data(), data.data(), Key::BINARY_LENGTH);
+  }
+
+  cpputils::unique_ref<BlockStore2> _baseBlockStore;
+  typename Cipher::EncryptionKey _encKey;
+
+  DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore2);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/ondisk/OnDiskBlockStore.h

@@ -4,7 +4,7 @@
 
 #include <boost/filesystem.hpp>
 #include "../../interface/helpers/BlockStoreWithRandomKeys.h"
-
+#include "../../interface/BlockStore2.h"
 #include <cpp-utils/macros.h>
 
 namespace blockstore {
@@ -16,7 +16,7 @@ public:
 
   boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
   boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
-  //TODO Can we make this faster by allowing to delete blocks by only having theiy Key? So we wouldn't have to load it first?
+  //TODO Can we make this faster by allowing to delete blocks by only having their Key? So we wouldn't have to load it first?
   void remove(const Key &key) override;
   uint64_t numBlocks() const override;
   uint64_t estimateNumFreeBytes() const override;

src/blockstore/implementations/ondisk/OnDiskBlockStore2.cpp

@@ -0,0 +1,7 @@
+#include "OnDiskBlockStore2.h"
+
+namespace blockstore {
+namespace ondisk {
+
+}
+}

src/blockstore/implementations/ondisk/OnDiskBlockStore2.h

@@ -0,0 +1,59 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE2_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE2_H_
+
+#include "../../interface/BlockStore2.h"
+#include <boost/filesystem/path.hpp>
+#include <cpp-utils/macros.h>
+#include <cpp-utils/pointer/unique_ref.h>
+#include "OnDiskBlockStore.h"
+
+namespace blockstore {
+namespace ondisk {
+
+//TODO Implement without basing on OnDiskBlockStore
+
+class OnDiskBlockStore2 final: public BlockStore2 {
+public:
+  explicit OnDiskBlockStore2(cpputils::unique_ref<OnDiskBlockStore> base)
+    : _base(std::move(base)) {}
+
+  boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override {
+    auto created = _base->tryCreate(key, data.copy());
+    if (boost::none == created) {
+      return boost::make_ready_future(false);
+    } else {
+      return boost::make_ready_future(true);
+    }
+  }
+
+  boost::future<bool> remove(const Key &key) override {
+    _base->remove(key);
+    return boost::make_ready_future(true);
+  }
+
+  boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override {
+    auto block = _base->load(key);
+    if (boost::none == block) {
+      return boost::make_ready_future<boost::optional<cpputils::Data>>(boost::none);
+    }
+    cpputils::Data data((*block)->size());
+    std::memcpy(data.data(), (*block)->data(), data.size());
+    return boost::make_ready_future<boost::optional<cpputils::Data>>(std::move(data));
+  }
+
+  boost::future<void> store(const Key &key, const cpputils::Data &data) override {
+    _base->overwrite(key, data.copy());
+    return boost::make_ready_future();
+  }
+
+private:
+  cpputils::unique_ref<OnDiskBlockStore> _base;
+
+  DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore2);
+};
+
+}
+}
+
+#endif

src/blockstore/implementations/versioncounting/VersionCountingBlockStore2.cpp

@@ -0,0 +1 @@
+#include "VersionCountingBlockStore2.h"

src/blockstore/implementations/versioncounting/VersionCountingBlockStore2.h

@@ -0,0 +1,127 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_VERSIONCOUNTING_VERSIONCOUNTINGBLOCKSTORE2_H_
+#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_VERSIONCOUNTING_VERSIONCOUNTINGBLOCKSTORE2_H_
+
+#include "../../interface/BlockStore2.h"
+#include <cpp-utils/macros.h>
+#include "KnownBlockVersions.h"
+
+namespace blockstore {
+namespace versioncounting {
+
+//TODO Format version headers
+
+class VersionCountingBlockStore2 final: public BlockStore2 {
+public:
+  VersionCountingBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId, bool missingBlockIsIntegrityViolation)
+      : _baseBlockStore(std::move(baseBlockStore)), _knownBlockVersions(integrityFilePath, myClientId), _missingBlockIsIntegrityViolation(missingBlockIsIntegrityViolation), _integrityViolationDetected(false) {
+  }
+
+  boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override {
+    _checkNoPastIntegrityViolations();
+    uint64_t version = blockStore->knownBlockVersions()->incrementVersion(key);
+    cpputils::Data dataWithHeader = _prependHeaderToData(blockStore->knownBlockVersions()->myClientId(), version, data);
+    return baseBlockStore_->tryCreate(key, dataWithHeader);
+  }
+
+  boost::future<bool> remove(const Key &key) override {
+    _checkNoPastIntegrityViolations();
+    _knownBlockVersions.markBlockAsDeleted(key);
+    return baseBlockStore->remove(key);
+  }
+
+  boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override {
+    _checkNoPastIntegrityViolations();
+    auto loaded = baseBlockStore_->load(key);
+    loaded.then([this, key] (boost::future<boost::optional<cpputils::Data>> loaded_) {
+      auto loaded = loaded_.get();
+      if (boost::none == loaded) {
+        if (_missingBlockIsIntegrityViolation && _knownBlockVersions.blockShouldExist(key)) {
+          integrityViolationDetected("A block that should exist wasn't found. Did an attacker delete it?");
+        }
+        return boost::none;
+      }
+      if (!_checkHeader(key, *loaded)) {
+        return boost::none;
+      }
+      return *loaded;
+    });
+  }
+
+  boost::future<void> store(const Key &key, const cpputils::Data &data) override {
+    _checkNoPastIntegrityViolations();
+    TODO Need to load first so it can see if the version number changed by another
+    THIS BUG IS ALSO IN THE NEXT BRANCH (i.e. without these changes here)
+    //...
+  }
+
+  static constexpr uint64_t VERSION_ZERO = 0;
+  static constexpr unsigned int CLIENTID_HEADER_OFFSET = sizeof(FORMAT_VERSION_HEADER);
+  static constexpr unsigned int VERSION_HEADER_OFFSET = sizeof(FORMAT_VERSION_HEADER) + sizeof(uint32_t);
+  static constexpr unsigned int HEADER_LENGTH = sizeof(FORMAT_VERSION_HEADER) + sizeof(uint32_t) + sizeof(VERSION_ZERO);
+
+private:
+
+  // This header is prepended to blocks to allow future versions to have compatibility.
+  static constexpr uint16_t FORMAT_VERSION_HEADER = 0;
+
+  cpputils::Data VersionCountingBlock::_prependHeaderToData(uint32_t myClientId, uint64_t version, const cpputils::Data &data) {
+    static_assert(HEADER_LENGTH == sizeof(FORMAT_VERSION_HEADER) + sizeof(myClientId) + sizeof(version), "Wrong header length");
+    cpputils::Data result(data.size() + HEADER_LENGTH);
+    std::memcpy(result.dataOffset(0), &FORMAT_VERSION_HEADER, sizeof(FORMAT_VERSION_HEADER));
+    std::memcpy(result.dataOffset(CLIENTID_HEADER_OFFSET), &myClientId, sizeof(myClientId));
+    std::memcpy(result.dataOffset(VERSION_HEADER_OFFSET), &version, sizeof(version));
+    std::memcpy((uint8_t*)result.dataOffset(HEADER_LENGTH), data.data(), data.size());
+    return result;
+  }
+
+  void _checkHeader(const Key &key, const cpputils::Data &data) {
+    _checkFormatHeader(data);
+    _checkVersionHeader(key, data);
+  }
+
+  void _checkFormatHeader(const cpputils::Data &data) {
+    if (*reinterpret_cast<decltype(FORMAT_VERSION_HEADER)*>(data.data()) != FORMAT_VERSION_HEADER) {
+      throw std::runtime_error("The versioned block has the wrong format. Was it created with a newer version of CryFS?");
+    }
+  }
+
+  void _checkVersionHeader(const Key &key, const cpputils::Data &data) {
+    uint32_t clientId;
+    std::memcpy(&clientId, _dataWithHeader.dataOffset(CLIENTID_HEADER_OFFSET), sizeof(clientId));
+
+    uint64_t version;
+    std::memcpy(&version, _dataWithHeader.dataOffset(VERSION_HEADER_OFFSET), sizeof(version));
+
+    if(!_knownBlockVersions.checkAndUpdateVersion(lastClientId, key, version)) {
+      integrityViolationDetected("The block version number is too low. Did an attacker try to roll back the block or to re-introduce a deleted block?");
+    }
+  }
+
+  void _checkNoPastIntegrityViolations() {
+    if (_integrityViolationDetected) {
+      throw std::runtime_error(string() +
+                               "There was an integrity violation detected. Preventing any further access to the file system. " +
+                               "If you want to reset the integrity data (i.e. accept changes made by a potential attacker), " +
+                               "please unmount the file system and delete the following file before re-mounting it: " +
+                               _knownBlockVersions.path().native());
+    }
+  }
+
+  void integrityViolationDetected(const string &reason) const {
+    _integrityViolationDetected = true;
+    throw IntegrityViolationError(reason);
+  }
+
+  cpputils::unique_ref<BlockStore2> _baseBlockStore;
+  KnownBlockVersions _knownBlockVersions;
+  const bool _missingBlockIsIntegrityViolation;
+  mutable bool _integrityViolationDetected;
+
+  DISALLOW_COPY_AND_ASSIGN(VersionCountingBlockStore2);
+};
+
+}
+}
+
+#endif

src/blockstore/interface/BlockStore2.h

@@ -0,0 +1,40 @@
+#pragma once
+#ifndef MESSMER_BLOCKSTORE_INTERFACE_BLOCKSTORE2_H_
+#define MESSMER_BLOCKSTORE_INTERFACE_BLOCKSTORE2_H_
+
+#include "Block.h"
+#include <string>
+#include <boost/optional.hpp>
+#include <cpp-utils/pointer/unique_ref.h>
+#include <cpp-utils/data/Data.h>
+#include <boost/thread/future.hpp>
+#include <cpp-utils/random/Random.h>
+
+namespace blockstore {
+
+class BlockStore2 {
+public:
+  virtual ~BlockStore2() {}
+
+  virtual boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) = 0;
+  virtual boost::future<bool> remove(const Key &key) = 0;
+
+  virtual boost::future<boost::optional<cpputils::Data>> load(const Key &key) const = 0;
+  virtual boost::future<void> store(const Key &key, const cpputils::Data &data) = 0;
+
+  boost::future<Key> create(cpputils::Data data) {
+    Key key = cpputils::Random::PseudoRandom().getFixedSize<Key::BINARY_LENGTH>();
+    boost::future<bool> successFuture = tryCreate(key, data);
+    return successFuture.then([this, key, data = std::move(data)] (boost::future<bool> success) mutable {
+      if (success.get()) {
+        return boost::make_ready_future<Key>(key);
+      } else {
+        return this->create(std::move(data));
+      }
+    });
+  }
+};
+
+}
+
+#endif

utils.cmake

@@ -57,11 +57,13 @@ function(target_add_boost TARGET)
         message(STATUS "Boost will be dynamically linked")
         set(Boost_USE_STATIC_LIBS OFF)
     endif(NOT DEFINED Boost_USE_STATIC_LIBS OR Boost_USE_STATIC_LIBS)
-    find_package(Boost 1.56.0
+    set(BOOST_THREAD_VERSION 4)
+    find_package(Boost 1.58.0
             REQUIRED
             COMPONENTS ${ARGN})
     target_include_directories(${TARGET} SYSTEM PUBLIC ${Boost_INCLUDE_DIRS})
     target_link_libraries(${TARGET} PUBLIC ${Boost_LIBRARIES})
+    target_compile_definitions(${TARGET} PUBLIC BOOST_THREAD_VERSION=4)
     if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
       # Also link to rt, because boost thread needs that.
       target_link_libraries(${TARGET} PUBLIC rt)