Actually wire up CryDevice to use new block stores

src/blockstore/implementations/inmemory/InMemoryBlockStore2.cpp

@@ -10,6 +10,7 @@ using std::lock_guard;
 using std::piecewise_construct;
 using std::make_tuple;
 using std::make_pair;
+using std::vector;
 using cpputils::Data;
 using cpputils::unique_ref;
 using cpputils::make_unique_ref;
@@ -25,11 +26,13 @@ InMemoryBlockStore2::InMemoryBlockStore2()
  : _blocks() {}
 
 future<bool> InMemoryBlockStore2::tryCreate(const Key &key, const Data &data) {
+  std::unique_lock<std::mutex> lock(_mutex);
   auto result = _blocks.insert(make_pair(key, data.copy()));
   return make_ready_future(result.second); // Return if insertion was successful (i.e. key didn't exist yet)
 }
 
 future<bool> InMemoryBlockStore2::remove(const Key &key) {
+  std::unique_lock<std::mutex> lock(_mutex);
   auto found = _blocks.find(key);
   if (found == _blocks.end()) {
     // Key not found
@@ -41,6 +44,7 @@ future<bool> InMemoryBlockStore2::remove(const Key &key) {
 }
 
 future<optional<Data>> InMemoryBlockStore2::load(const Key &key) const {
+  std::unique_lock<std::mutex> lock(_mutex);
   auto found = _blocks.find(key);
   if (found == _blocks.end()) {
     return make_ready_future(optional<Data>(none));
@@ -49,6 +53,7 @@ future<optional<Data>> InMemoryBlockStore2::load(const Key &key) const {
 }
 
 future<void> InMemoryBlockStore2::store(const Key &key, const Data &data) {
+  std::unique_lock<std::mutex> lock(_mutex);
   auto found = _blocks.find(key);
   if (found == _blocks.end()) {
     return tryCreate(key, data).then([] (future<bool> success) {
@@ -63,6 +68,7 @@ future<void> InMemoryBlockStore2::store(const Key &key, const Data &data) {
 }
 
 uint64_t InMemoryBlockStore2::numBlocks() const {
+  std::unique_lock<std::mutex> lock(_mutex);
   return _blocks.size();
 }
 
@@ -74,9 +80,20 @@ uint64_t InMemoryBlockStore2::blockSizeFromPhysicalBlockSize(uint64_t blockSize)
   return blockSize;
 }
 
-void InMemoryBlockStore2::forEachBlock(std::function<void (const Key &)> callback) const {
+vector<Key> InMemoryBlockStore2::_allBlockKeys() const {
+  std::unique_lock<std::mutex> lock(_mutex);
+  vector<Key> result;
+  result.reserve(_blocks.size());
   for (const auto &entry : _blocks) {
-    callback(entry.first);
+    result.push_back(entry.first);
+  }
+  return result;
+}
+
+void InMemoryBlockStore2::forEachBlock(std::function<void (const Key &)> callback) const {
+  auto keys = _allBlockKeys();
+  for (const auto &key : keys) {
+    callback(key);
   }
 }
 

src/blockstore/implementations/inmemory/InMemoryBlockStore2.h

@@ -23,7 +23,10 @@ public:
   void forEachBlock(std::function<void (const Key &)> callback) const override;
 
 private:
+  std::vector<Key> _allBlockKeys() const;
+
   std::unordered_map<Key, cpputils::Data> _blocks;
+  mutable std::mutex _mutex;
 
   DISALLOW_COPY_AND_ASSIGN(InMemoryBlockStore2);
 };

src/blockstore/implementations/versioncounting/VersionCountingBlock.cpp

@@ -7,17 +7,5 @@ namespace blockstore {
         constexpr unsigned int VersionCountingBlock::HEADER_LENGTH;
         constexpr uint16_t VersionCountingBlock::FORMAT_VERSION_HEADER;
         constexpr uint64_t VersionCountingBlock::VERSION_ZERO;
-
-#ifndef CRYFS_NO_COMPATIBILITY
-        void VersionCountingBlock::migrateFromBlockstoreWithoutVersionNumbers(cpputils::unique_ref<Block> baseBlock, KnownBlockVersions *knownBlockVersions) {
-            uint64_t version = knownBlockVersions->incrementVersion(baseBlock->key());
-
-            cpputils::Data data(baseBlock->size());
-            std::memcpy(data.data(), baseBlock->data(), data.size());
-            cpputils::Data dataWithHeader = _prependHeaderToData(knownBlockVersions->myClientId(), version, std::move(data));
-            baseBlock->resize(dataWithHeader.size());
-            baseBlock->write(dataWithHeader.data(), 0, dataWithHeader.size());
-        }
-#endif
     }
 }

src/blockstore/implementations/versioncounting/VersionCountingBlock.h

@@ -45,10 +45,6 @@ public:
 
   cpputils::unique_ref<Block> releaseBlock();
 
-#ifndef CRYFS_NO_COMPATIBILITY
-  static void migrateFromBlockstoreWithoutVersionNumbers(cpputils::unique_ref<Block> baseBlock, KnownBlockVersions *knownBlockVersions);
-#endif
-
 private:
   VersionCountingBlockStore *_blockStore;
   cpputils::unique_ref<Block> _baseBlock;

src/blockstore/implementations/versioncounting/VersionCountingBlockStore.cpp

@@ -99,18 +99,5 @@ namespace blockstore {
             }
         }
 
-#ifndef CRYFS_NO_COMPATIBILITY
-        void VersionCountingBlockStore::migrateFromBlockstoreWithoutVersionNumbers(BlockStore *baseBlockStore, const bf::path &integrityFilePath, uint32_t myClientId) {
-            std::cout << "Migrating file system for integrity features. Please don't interrupt this process. This can take a while..." << std::flush;
-            KnownBlockVersions knownBlockVersions(integrityFilePath, myClientId);
-            baseBlockStore->forEachBlock([&baseBlockStore, &knownBlockVersions] (const Key &key) {
-                auto block =  baseBlockStore->load(key);
-                ASSERT(block != none, "Couldn't load block for migration");
-                VersionCountingBlock::migrateFromBlockstoreWithoutVersionNumbers(std::move(*block), &knownBlockVersions);
-            });
-            std::cout << "done" << std::endl;
-        }
-#endif
-
     }
 }

src/blockstore/implementations/versioncounting/VersionCountingBlockStore.h

@@ -28,10 +28,6 @@ public:
   void integrityViolationDetected(const std::string &reason) const;
   KnownBlockVersions *knownBlockVersions();
 
-#ifndef CRYFS_NO_COMPATIBILITY
-  static void migrateFromBlockstoreWithoutVersionNumbers(BlockStore *baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId);
-#endif
-
 private:
   cpputils::unique_ref<BlockStore> _baseBlockStore;
   KnownBlockVersions _knownBlockVersions;

src/blockstore/implementations/versioncounting/VersionCountingBlockStore2.cpp

@@ -1,4 +1,6 @@
+#include <blockstore/interface/BlockStore2.h>
 #include "VersionCountingBlockStore2.h"
+#include "KnownBlockVersions.h"
 
 using cpputils::Data;
 using cpputils::unique_ref;
@@ -6,6 +8,7 @@ using std::string;
 using boost::optional;
 using boost::none;
 using boost::future;
+using namespace cpputils::logging;
 
 namespace blockstore {
 namespace versioncounting {
@@ -145,5 +148,29 @@ void VersionCountingBlockStore2::forEachBlock(std::function<void (const Key &)>
   }
 }
 
+#ifndef CRYFS_NO_COMPATIBILITY
+void VersionCountingBlockStore2::migrateFromBlockstoreWithoutVersionNumbers(BlockStore2 *baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId) {
+  std::cout << "Migrating file system for integrity features. Please don't interrupt this process. This can take a while..." << std::flush;
+  KnownBlockVersions knownBlockVersions(integrityFilePath, myClientId);
+  baseBlockStore->forEachBlock([&baseBlockStore, &knownBlockVersions] (const Key &key) {
+    migrateBlockFromBlockstoreWithoutVersionNumbers(baseBlockStore, key, &knownBlockVersions);
+  });
+  std::cout << "done" << std::endl;
+}
+
+void VersionCountingBlockStore2::migrateBlockFromBlockstoreWithoutVersionNumbers(blockstore::BlockStore2* baseBlockStore, const blockstore::Key& key, KnownBlockVersions *knownBlockVersions) {
+  uint64_t version = knownBlockVersions->incrementVersion(key);
+
+  auto data_ = baseBlockStore->load(key).get(); // TODO this is blocking
+  if (data_ == boost::none) {
+    LOG(WARN, "Block not found, but was returned from forEachBlock before");
+    return;
+  }
+  cpputils::Data data = std::move(*data_);
+  cpputils::Data dataWithHeader = _prependHeaderToData(knownBlockVersions->myClientId(), version, std::move(data));
+  baseBlockStore->store(key, std::move(dataWithHeader)).wait(); // TODO This is blocking
+}
+#endif
+
 }
 }

src/blockstore/implementations/versioncounting/VersionCountingBlockStore2.h

@@ -35,9 +35,14 @@ public:
   static constexpr unsigned int VERSION_HEADER_OFFSET = sizeof(FORMAT_VERSION_HEADER) + sizeof(uint32_t);
   static constexpr unsigned int HEADER_LENGTH = sizeof(FORMAT_VERSION_HEADER) + sizeof(uint32_t) + sizeof(VERSION_ZERO);
 
+#ifndef CRYFS_NO_COMPATIBILITY
+  static void migrateFromBlockstoreWithoutVersionNumbers(BlockStore2 *baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId);
+  static void migrateBlockFromBlockstoreWithoutVersionNumbers(BlockStore2* baseBlockStore, const blockstore::Key& key, KnownBlockVersions *knownBlockVersions);
+#endif
+
 private:
 
-  cpputils::Data _prependHeaderToData(uint32_t myClientId, uint64_t version, const cpputils::Data &data);
+  static cpputils::Data _prependHeaderToData(uint32_t myClientId, uint64_t version, const cpputils::Data &data);
   void _checkHeader(const Key &key, const cpputils::Data &data) const;
   void _checkFormatHeader(const cpputils::Data &data) const;
   void _checkVersionHeader(const Key &key, const cpputils::Data &data) const;

src/cryfs-cli/Cli.cpp

@@ -1,6 +1,6 @@
 #include "Cli.h"
 
-#include <blockstore/implementations/ondisk/OnDiskBlockStore.h>
+#include <blockstore/implementations/ondisk/OnDiskBlockStore2.h>
 #include <blockstore/implementations/inmemory/InMemoryBlockStore.h>
 #include <blockstore/implementations/inmemory/InMemoryBlock.h>
 #include <cmath>
@@ -32,7 +32,7 @@ using namespace cryfs;
 namespace bf = boost::filesystem;
 using namespace cpputils::logging;
 
-using blockstore::ondisk::OnDiskBlockStore;
+using blockstore::ondisk::OnDiskBlockStore2;
 using blockstore::inmemory::InMemoryBlockStore;
 using program_options::ProgramOptions;
 
@@ -229,7 +229,7 @@ namespace cryfs {
 
     void Cli::_runFilesystem(const ProgramOptions &options) {
         try {
-            auto blockStore = make_unique_ref<OnDiskBlockStore>(options.baseDir());
+            auto blockStore = make_unique_ref<OnDiskBlockStore2>(options.baseDir());
             auto config = _loadOrCreateConfig(options);
             CryDevice device(std::move(config.configFile), std::move(blockStore), config.myClientId);
             _sanityCheckFilesystem(&device);

src/cryfs/config/CryCipher.cpp

@@ -1,7 +1,7 @@
 #include "CryCipher.h"
 
 #include <cpp-utils/crypto/symmetric/ciphers.h>
-#include <blockstore/implementations/encrypted/EncryptedBlockStore.h>
+#include <blockstore/implementations/encrypted/EncryptedBlockStore2.h>
 #include "crypto/inner/ConcreteInnerEncryptor.h"
 
 using std::vector;
@@ -9,12 +9,12 @@ using std::string;
 using cpputils::unique_ref;
 using cpputils::make_unique_ref;
 using cpputils::FixedSizeData;
-using blockstore::BlockStore;
+using blockstore::BlockStore2;
 using std::shared_ptr;
 using std::make_shared;
 using boost::optional;
 using boost::none;
-using blockstore::encrypted::EncryptedBlockStore;
+using blockstore::encrypted::EncryptedBlockStore2;
 
 using namespace cryfs;
 using namespace cpputils;
@@ -39,8 +39,8 @@ public:
         return _warning;
     }
 
-    unique_ref<BlockStore> createEncryptedBlockstore(unique_ref<BlockStore> baseBlockStore, const string &encKey) const override {
+    unique_ref<BlockStore2> createEncryptedBlockstore(unique_ref<BlockStore2> baseBlockStore, const string &encKey) const override {
-        return make_unique_ref<EncryptedBlockStore<Cipher>>(std::move(baseBlockStore), Cipher::EncryptionKey::FromString(encKey));
+        return make_unique_ref<EncryptedBlockStore2<Cipher>>(std::move(baseBlockStore), Cipher::EncryptionKey::FromString(encKey));
     }
 
     string createKey(cpputils::RandomGenerator &randomGenerator) const override {

src/cryfs/config/CryCipher.h

@@ -5,7 +5,7 @@
 #include <vector>
 #include <string>
 #include <cpp-utils/pointer/unique_ref.h>
-#include <blockstore/interface/BlockStore.h>
+#include <blockstore/interface/BlockStore2.h>
 #include <cpp-utils/random/RandomGenerator.h>
 #include "crypto/inner/InnerEncryptor.h"
 
@@ -36,7 +36,7 @@ public:
 
     virtual std::string cipherName() const = 0;
     virtual const boost::optional<std::string> &warning() const = 0;
-    virtual cpputils::unique_ref<blockstore::BlockStore> createEncryptedBlockstore(cpputils::unique_ref<blockstore::BlockStore> baseBlockStore, const std::string &encKey) const = 0;
+    virtual cpputils::unique_ref<blockstore::BlockStore2> createEncryptedBlockstore(cpputils::unique_ref<blockstore::BlockStore2> baseBlockStore, const std::string &encKey) const = 0;
     virtual std::string createKey(cpputils::RandomGenerator &randomGenerator) const = 0;
     virtual cpputils::unique_ref<InnerEncryptor> createInnerConfigEncryptor(const cpputils::FixedSizeData<CryCiphers::MAX_KEY_SIZE> &key) const = 0;
 };

src/cryfs/filesystem/CryDevice.cpp

@@ -10,13 +10,16 @@
 #include <fspp/fuse/FuseErrnoException.h>
 #include <blobstore/implementations/onblocks/BlobStoreOnBlocks.h>
 #include <blobstore/implementations/onblocks/BlobOnBlocks.h>
-#include <blockstore/implementations/encrypted/EncryptedBlockStore.h>
+#include <blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h>
+#include <blockstore/implementations/encrypted/EncryptedBlockStore2.h>
 #include <blockstore/implementations/versioncounting/VersionCountingBlock.h>
+#include <blockstore/implementations/versioncounting/VersionCountingBlockStore2.h>
 #include "parallelaccessfsblobstore/ParallelAccessFsBlobStore.h"
 #include "cachingfsblobstore/CachingFsBlobStore.h"
 #include "../config/CryCipher.h"
 #include <cpp-utils/system/homedir.h>
 #include <gitversion/VersionCompare.h>
+#include <blockstore/interface/BlockStore2.h>
 #include "cryfs/localstate/MyClientId.h"
 #include "cryfs/localstate/LocalStateDir.h"
 
@@ -27,14 +30,15 @@ using fspp::fuse::CHECK_RETVAL;
 using fspp::fuse::FuseErrnoException;
 
 using blockstore::BlockStore;
+using blockstore::BlockStore2;
 using blockstore::Key;
-using blockstore::encrypted::EncryptedBlockStore;
+using blockstore::encrypted::EncryptedBlockStore2;
 using blobstore::BlobStore;
+using blockstore::lowtohighlevel::LowToHighLevelBlockStore;
 using blobstore::onblocks::BlobStoreOnBlocks;
 using blobstore::onblocks::BlobOnBlocks;
 using blockstore::caching::CachingBlockStore;
-using blockstore::versioncounting::VersionCountingBlockStore;
+using blockstore::versioncounting::VersionCountingBlockStore2;
-using blockstore::versioncounting::VersionCountingBlock;
 using gitversion::VersionCompare;
 using cpputils::unique_ref;
 using cpputils::make_unique_ref;
@@ -54,13 +58,13 @@ namespace bf = boost::filesystem;
 
 namespace cryfs {
 
-CryDevice::CryDevice(CryConfigFile configFile, unique_ref<BlockStore> blockStore, uint32_t myClientId)
+CryDevice::CryDevice(CryConfigFile configFile, unique_ref<BlockStore2> blockStore, uint32_t myClientId)
 : _fsBlobStore(CreateFsBlobStore(std::move(blockStore), &configFile, myClientId)),
   _rootKey(GetOrCreateRootKey(&configFile)),
   _onFsAction() {
 }
 
-unique_ref<parallelaccessfsblobstore::ParallelAccessFsBlobStore> CryDevice::CreateFsBlobStore(unique_ref<BlockStore> blockStore, CryConfigFile *configFile, uint32_t myClientId) {
+unique_ref<parallelaccessfsblobstore::ParallelAccessFsBlobStore> CryDevice::CreateFsBlobStore(unique_ref<BlockStore2> blockStore, CryConfigFile *configFile, uint32_t myClientId) {
   auto blobStore = CreateBlobStore(std::move(blockStore), configFile, myClientId);
 
 #ifndef CRYFS_NO_COMPATIBILITY
@@ -86,32 +90,34 @@ unique_ref<fsblobstore::FsBlobStore> CryDevice::MigrateOrCreateFsBlobStore(uniqu
 }
 #endif
 
-unique_ref<blobstore::BlobStore> CryDevice::CreateBlobStore(unique_ref<BlockStore> blockStore, CryConfigFile *configFile, uint32_t myClientId) {
+unique_ref<blobstore::BlobStore> CryDevice::CreateBlobStore(unique_ref<BlockStore2> blockStore, CryConfigFile *configFile, uint32_t myClientId) {
   auto versionCountingEncryptedBlockStore = CreateVersionCountingEncryptedBlockStore(std::move(blockStore), configFile, myClientId);
   // Create versionCountingEncryptedBlockStore not in the same line as BlobStoreOnBlocks, because it can modify BlocksizeBytes
   // in the configFile and therefore has to be run before the second parameter to the BlobStoreOnBlocks parameter is evaluated.
   return make_unique_ref<BlobStoreOnBlocks>(
      make_unique_ref<CachingBlockStore>(
+         make_unique_ref<LowToHighLevelBlockStore>(
              std::move(versionCountingEncryptedBlockStore)
+         )
      ),
      configFile->config()->BlocksizeBytes());
 }
 
-unique_ref<BlockStore> CryDevice::CreateVersionCountingEncryptedBlockStore(unique_ref<BlockStore> blockStore, CryConfigFile *configFile, uint32_t myClientId) {
+unique_ref<BlockStore2> CryDevice::CreateVersionCountingEncryptedBlockStore(unique_ref<BlockStore2> blockStore, CryConfigFile *configFile, uint32_t myClientId) {
   auto encryptedBlockStore = CreateEncryptedBlockStore(*configFile->config(), std::move(blockStore));
   auto statePath = LocalStateDir::forFilesystemId(configFile->config()->FilesystemId());
   auto integrityFilePath = statePath / "integritydata";
 
 #ifndef CRYFS_NO_COMPATIBILITY
   if (!configFile->config()->HasVersionNumbers()) {
-    VersionCountingBlockStore::migrateFromBlockstoreWithoutVersionNumbers(encryptedBlockStore.get(), integrityFilePath, myClientId);
+    VersionCountingBlockStore2::migrateFromBlockstoreWithoutVersionNumbers(encryptedBlockStore.get(), integrityFilePath, myClientId);
-    configFile->config()->SetBlocksizeBytes(configFile->config()->BlocksizeBytes() + VersionCountingBlock::HEADER_LENGTH);
+    configFile->config()->SetBlocksizeBytes(configFile->config()->BlocksizeBytes() + VersionCountingBlockStore2::HEADER_LENGTH);
     configFile->config()->SetHasVersionNumbers(true);
     configFile->save();
   }
 #endif
 
-  return make_unique_ref<VersionCountingBlockStore>(std::move(encryptedBlockStore), integrityFilePath, myClientId, false);
+  return make_unique_ref<VersionCountingBlockStore2>(std::move(encryptedBlockStore), integrityFilePath, myClientId, false);
 }
 
 Key CryDevice::CreateRootBlobAndReturnKey() {
@@ -294,7 +300,7 @@ Key CryDevice::GetOrCreateRootKey(CryConfigFile *configFile) {
   return Key::FromString(root_key);
 }
 
-cpputils::unique_ref<blockstore::BlockStore> CryDevice::CreateEncryptedBlockStore(const CryConfig &config, unique_ref<BlockStore> baseBlockStore) {
+cpputils::unique_ref<blockstore::BlockStore2> CryDevice::CreateEncryptedBlockStore(const CryConfig &config, unique_ref<BlockStore2> baseBlockStore) {
   //TODO Test that CryFS is using the specified cipher
   return CryCiphers::find(config.Cipher()).createEncryptedBlockstore(std::move(baseBlockStore), config.EncryptionKey());
 }

src/cryfs/filesystem/CryDevice.h

@@ -3,6 +3,7 @@
 #define MESSMER_CRYFS_FILESYSTEM_CRYDEVICE_H_
 
 #include <blockstore/interface/BlockStore.h>
+#include <blockstore/interface/BlockStore2.h>
 #include "../config/CryConfigFile.h"
 
 #include <boost/filesystem.hpp>
@@ -17,7 +18,7 @@ namespace cryfs {
 
 class CryDevice final: public fspp::Device {
 public:
-  CryDevice(CryConfigFile config, cpputils::unique_ref<blockstore::BlockStore> blockStore, uint32_t myClientId);
+  CryDevice(CryConfigFile config, cpputils::unique_ref<blockstore::BlockStore2> blockStore, uint32_t myClientId);
 
   void statfs(const boost::filesystem::path &path, struct ::statvfs *fsstat) override;
 
@@ -52,13 +53,13 @@ private:
 
   blockstore::Key GetOrCreateRootKey(CryConfigFile *config);
   blockstore::Key CreateRootBlobAndReturnKey();
-  static cpputils::unique_ref<parallelaccessfsblobstore::ParallelAccessFsBlobStore> CreateFsBlobStore(cpputils::unique_ref<blockstore::BlockStore> blockStore, CryConfigFile *configFile, uint32_t myClientId);
+  static cpputils::unique_ref<parallelaccessfsblobstore::ParallelAccessFsBlobStore> CreateFsBlobStore(cpputils::unique_ref<blockstore::BlockStore2> blockStore, CryConfigFile *configFile, uint32_t myClientId);
 #ifndef CRYFS_NO_COMPATIBILITY
   static cpputils::unique_ref<fsblobstore::FsBlobStore> MigrateOrCreateFsBlobStore(cpputils::unique_ref<blobstore::BlobStore> blobStore, CryConfigFile *configFile);
 #endif
-  static cpputils::unique_ref<blobstore::BlobStore> CreateBlobStore(cpputils::unique_ref<blockstore::BlockStore> blockStore, CryConfigFile *configFile, uint32_t myClientId);
+  static cpputils::unique_ref<blobstore::BlobStore> CreateBlobStore(cpputils::unique_ref<blockstore::BlockStore2> blockStore, CryConfigFile *configFile, uint32_t myClientId);
-  static cpputils::unique_ref<blockstore::BlockStore> CreateVersionCountingEncryptedBlockStore(cpputils::unique_ref<blockstore::BlockStore> blockStore, CryConfigFile *configFile, uint32_t myClientId);
+  static cpputils::unique_ref<blockstore::BlockStore2> CreateVersionCountingEncryptedBlockStore(cpputils::unique_ref<blockstore::BlockStore2> blockStore, CryConfigFile *configFile, uint32_t myClientId);
-  static cpputils::unique_ref<blockstore::BlockStore> CreateEncryptedBlockStore(const CryConfig &config, cpputils::unique_ref<blockstore::BlockStore> baseBlockStore);
+  static cpputils::unique_ref<blockstore::BlockStore2> CreateEncryptedBlockStore(const CryConfig &config, cpputils::unique_ref<blockstore::BlockStore2> baseBlockStore);
 
   struct BlobWithParent {
       cpputils::unique_ref<parallelaccessfsblobstore::FsBlobRef> blob;

src/stats/main.cpp

@@ -1,7 +1,8 @@
 #include <iostream>
 #include <boost/filesystem.hpp>
 #include <cryfs/config/CryConfigFile.h>
-#include <blockstore/implementations/ondisk/OnDiskBlockStore.h>
+#include <blockstore/implementations/ondisk/OnDiskBlockStore2.h>
+#include <blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h>
 #include <blobstore/implementations/onblocks/datanodestore/DataNodeStore.h>
 #include <blobstore/implementations/onblocks/datanodestore/DataNode.h>
 #include <blobstore/implementations/onblocks/datanodestore/DataInnerNode.h>
@@ -18,6 +19,7 @@ using namespace cryfs;
 using namespace cpputils;
 using namespace blockstore;
 using namespace blockstore::ondisk;
+using namespace blockstore::lowtohighlevel;
 using namespace blobstore::onblocks;
 using namespace blobstore::onblocks::datanodestore;
 using namespace cryfs::fsblobstore;
@@ -37,9 +39,10 @@ void printNode(unique_ref<DataNode> node) {
 }
 
 set<Key> _getBlockstoreUnaccountedBlocks(const CryConfig &config) {
-    auto onDiskBlockStore = make_unique_ref<OnDiskBlockStore>("/home/heinzi/basedir");
+    auto onDiskBlockStore = make_unique_ref<OnDiskBlockStore2>("/home/heinzi/basedir");
     auto encryptedBlockStore = CryCiphers::find(config.Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config.EncryptionKey());
-    auto nodeStore = make_unique_ref<DataNodeStore>(std::move(encryptedBlockStore), config.BlocksizeBytes());
+    auto highLevelBlockStore = make_unique_ref<LowToHighLevelBlockStore>(std::move(encryptedBlockStore));
+    auto nodeStore = make_unique_ref<DataNodeStore>(std::move(highLevelBlockStore), config.BlocksizeBytes());
     std::set<Key> unaccountedBlocks;
     uint32_t numBlocks = nodeStore->numNodes();
     uint32_t i = 0;
@@ -75,9 +78,10 @@ set<Key> _getBlockstoreUnaccountedBlocks(const CryConfig &config) {
 }
 
 set<Key> _getBlocksReferencedByDirEntries(const CryConfig &config) {
-    auto onDiskBlockStore = make_unique_ref<OnDiskBlockStore>("/home/heinzi/basedir");
+    auto onDiskBlockStore = make_unique_ref<OnDiskBlockStore2>("/home/heinzi/basedir");
     auto encryptedBlockStore = CryCiphers::find(config.Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config.EncryptionKey());
-    auto fsBlobStore = make_unique_ref<FsBlobStore>(make_unique_ref<BlobStoreOnBlocks>(std::move(encryptedBlockStore), config.BlocksizeBytes()));
+    auto highLevelBlockStore = make_unique_ref<LowToHighLevelBlockStore>(std::move(encryptedBlockStore));
+    auto fsBlobStore = make_unique_ref<FsBlobStore>(make_unique_ref<BlobStoreOnBlocks>(std::move(highLevelBlockStore), config.BlocksizeBytes()));
     set<Key> blocksReferencedByDirEntries;
     uint32_t numBlocks = fsBlobStore->numBlocks();
     uint32_t i = 0;
@@ -120,9 +124,10 @@ int main() {
 
     cout << "\nCalculate statistics" << endl;
 
-    auto onDiskBlockStore = make_unique_ref<OnDiskBlockStore>("/home/heinzi/basedir");
+    auto onDiskBlockStore = make_unique_ref<OnDiskBlockStore2>("/home/heinzi/basedir");
     auto encryptedBlockStore = CryCiphers::find(config->config()->Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config->config()->EncryptionKey());
-    auto nodeStore = make_unique_ref<DataNodeStore>(std::move(encryptedBlockStore), config->config()->BlocksizeBytes());
+    auto highLevelBlockStore = make_unique_ref<LowToHighLevelBlockStore>(std::move(encryptedBlockStore));
+    auto nodeStore = make_unique_ref<DataNodeStore>(std::move(highLevelBlockStore), config->config()->BlocksizeBytes());
 
     uint32_t numUnaccountedBlocks = unaccountedBlocks.size();
     uint32_t numLeaves = 0;

test/cryfs/config/CryCipherTest.cpp

@@ -3,14 +3,14 @@
 #include <cryfs/config/CryCipher.h>
 #include <cpp-utils/crypto/symmetric/ciphers.h>
 #include <cpp-utils/pointer/unique_ref_boost_optional_gtest_workaround.h>
-#include <blockstore/implementations/testfake/FakeBlockStore.h>
+#include <blockstore/implementations/inmemory/InMemoryBlockStore2.h>
-#include <blockstore/implementations/encrypted/EncryptedBlockStore.h>
+#include <blockstore/implementations/encrypted/EncryptedBlockStore2.h>
 #include <cpp-utils/data/DataFixture.h>
 #include <cpp-utils/random/Random.h>
 
 using namespace cryfs;
 using namespace blockstore::encrypted;
-using namespace blockstore::testfake;
+using namespace blockstore::inmemory;
 using namespace blockstore;
 
 using std::initializer_list;
@@ -50,28 +50,25 @@ public:
     }
 
     Data _encryptUsingEncryptedBlockStoreWithCipher(const CryCipher &cipher, const std::string &encKey, const blockstore::Key &key, Data data) {
-        unique_ref<FakeBlockStore> _baseStore = make_unique_ref<FakeBlockStore>();
+        unique_ref<InMemoryBlockStore2> _baseStore = make_unique_ref<InMemoryBlockStore2>();
-        FakeBlockStore *baseStore = _baseStore.get();
+        InMemoryBlockStore2 *baseStore = _baseStore.get();
-        unique_ref<BlockStore> encryptedStore = cipher.createEncryptedBlockstore(std::move(_baseStore), encKey);
+        unique_ref<BlockStore2> encryptedStore = cipher.createEncryptedBlockstore(std::move(_baseStore), encKey);
-        auto created = encryptedStore->tryCreate(key, std::move(data));
+        bool created = encryptedStore->tryCreate(key, std::move(data)).get();
-        EXPECT_NE(none, created);
+        EXPECT_TRUE(created);
         return _loadBlock(baseStore, key);
     }
 
     template<class Cipher>
     Data _decryptUsingEncryptedBlockStoreWithCipher(const std::string &encKey, const blockstore::Key &key, Data data) {
-        unique_ref<FakeBlockStore> baseStore = make_unique_ref<FakeBlockStore>();
+        unique_ref<InMemoryBlockStore2> baseStore = make_unique_ref<InMemoryBlockStore2>();
-        auto created = baseStore->tryCreate(key, std::move(data));
+        bool created = baseStore->tryCreate(key, std::move(data)).get();
-        EXPECT_NE(none, created);
+        EXPECT_TRUE(created);
-        EncryptedBlockStore<Cipher> encryptedStore(std::move(baseStore), Cipher::EncryptionKey::FromString(encKey));
+        EncryptedBlockStore2<Cipher> encryptedStore(std::move(baseStore), Cipher::EncryptionKey::FromString(encKey));
         return _loadBlock(&encryptedStore, key);
     }
 
-    Data _loadBlock(BlockStore *store, const blockstore::Key &key) {
+    Data _loadBlock(BlockStore2 *store, const blockstore::Key &key) {
-        auto block = store->load(key).value();
+        return store->load(key).get().value();
-        Data data(block->size());
-        std::memcpy(data.data(), block->data(), block->size());
-        return data;
     }
 };
 

test/cryfs/filesystem/CryFsTest.cpp

@@ -2,7 +2,7 @@
 #include <cpp-utils/tempfile/TempDir.h>
 #include <cpp-utils/tempfile/TempFile.h>
 #include <cpp-utils/pointer/cast.h>
-#include <blockstore/implementations/ondisk/OnDiskBlockStore.h>
+#include <blockstore/implementations/ondisk/OnDiskBlockStore2.h>
 #include <cryfs/filesystem/CryDevice.h>
 #include <cryfs/filesystem/CryDir.h>
 #include <cryfs/filesystem/CryFile.h>
@@ -30,7 +30,7 @@ using cpputils::SCrypt;
 using cpputils::Data;
 using cpputils::system::FakeHomeDirectoryRAII;
 using cpputils::NoninteractiveConsole;
-using blockstore::ondisk::OnDiskBlockStore;
+using blockstore::ondisk::OnDiskBlockStore2;
 using boost::none;
 
 namespace bf = boost::filesystem;
@@ -46,8 +46,8 @@ public:
     return CryConfigLoader(make_shared<NoninteractiveConsole>(mockConsole()), Random::PseudoRandom(), SCrypt::TestSettings, askPassword, askPassword, none, none, none).loadOrCreate(config.path()).value().configFile;
   }
 
-  unique_ref<OnDiskBlockStore> blockStore() {
+  unique_ref<OnDiskBlockStore2> blockStore() {
-    return make_unique_ref<OnDiskBlockStore>(rootdir.path());
+    return make_unique_ref<OnDiskBlockStore2>(rootdir.path());
   }
 
   TempDir rootdir;

test/cryfs/filesystem/FileSystemTest.cpp

@@ -1,4 +1,4 @@
-#include <blockstore/implementations/testfake/FakeBlockStore.h>
+#include <blockstore/implementations/inmemory/InMemoryBlockStore2.h>
 #include <fspp/fstest/FsTest.h>
 #include <cpp-utils/tempfile/TempFile.h>
 #include <cpp-utils/io/NoninteractiveConsole.h>
@@ -17,7 +17,7 @@ using ::testing::Return;
 using ::testing::_;
 using boost::none;
 using std::make_shared;
-using blockstore::testfake::FakeBlockStore;
+using blockstore::inmemory::InMemoryBlockStore2;
 
 using namespace cryfs;
 
@@ -28,7 +28,7 @@ public:
   : configFile(false) {}
 
   unique_ref<Device> createDevice() override {
-    auto blockStore = cpputils::make_unique_ref<FakeBlockStore>();
+    auto blockStore = cpputils::make_unique_ref<InMemoryBlockStore2>();
     auto askPassword = [] {return "mypassword";};
     auto config = CryConfigLoader(make_shared<NoninteractiveConsole>(mockConsole()), Random::PseudoRandom(), SCrypt::TestSettings, askPassword, askPassword, none, none, none)
             .loadOrCreate(configFile.path()).value();

test/cryfs/filesystem/testutils/CryTestBase.h

@@ -2,7 +2,7 @@
 #define MESSMER_CRYFS_TEST_CRYFS_FILESYSTEM_CRYTESTBASE_H
 
 #include <cryfs/filesystem/CryDevice.h>
-#include <blockstore/implementations/testfake/FakeBlockStore.h>
+#include <blockstore/implementations/inmemory/InMemoryBlockStore2.h>
 #include <cpp-utils/tempfile/TempFile.h>
 #include <cpp-utils/crypto/kdf/Scrypt.h>
 #include "../../testutils/TestWithFakeHomeDirectory.h"
@@ -10,7 +10,7 @@
 class CryTestBase : public TestWithFakeHomeDirectory {
 public:
     CryTestBase(): _configFile(false), _device(nullptr) {
-        auto fakeBlockStore = cpputils::make_unique_ref<blockstore::testfake::FakeBlockStore>();
+        auto fakeBlockStore = cpputils::make_unique_ref<blockstore::inmemory::InMemoryBlockStore2>();
         _device = std::make_unique<cryfs::CryDevice>(configFile(), std::move(fakeBlockStore), 0x12345678);
     }