Implement migration from file systems without version numbers

src/blockstore/implementations/versioncounting/VersionCountingBlock.cpp

@@ -7,5 +7,17 @@ namespace blockstore {
         constexpr unsigned int VersionCountingBlock::HEADER_LENGTH;
         constexpr uint16_t VersionCountingBlock::FORMAT_VERSION_HEADER;
         constexpr uint64_t VersionCountingBlock::VERSION_ZERO;
+
+#ifndef CRYFS_NO_COMPATIBILITY
+        void VersionCountingBlock::migrateFromBlockstoreWithoutVersionNumbers(cpputils::unique_ref<Block> baseBlock, KnownBlockVersions *knownBlockVersions) {
+            uint64_t version = knownBlockVersions->incrementVersion(baseBlock->key(), VERSION_ZERO);
+
+            cpputils::Data data(baseBlock->size());
+            std::memcpy(data.data(), baseBlock->data(), data.size());
+            cpputils::Data dataWithHeader = _prependHeaderToData(knownBlockVersions->myClientId(), version, std::move(data));
+            baseBlock->resize(dataWithHeader.size());
+            baseBlock->write(dataWithHeader.data(), 0, dataWithHeader.size());
+        }
+#endif
     }
 }

src/blockstore/implementations/versioncounting/VersionCountingBlock.h

@@ -44,6 +44,10 @@ public:
   uint64_t version() const;
   cpputils::unique_ref<Block> releaseBlock();
 
+#ifndef CRYFS_NO_COMPATIBILITY
+  static void migrateFromBlockstoreWithoutVersionNumbers(cpputils::unique_ref<Block> baseBlock, KnownBlockVersions *knownBlockVersions);
+#endif
+
 private:
   KnownBlockVersions *_knownBlockVersions;
   cpputils::unique_ref<Block> _baseBlock;

src/blockstore/implementations/versioncounting/VersionCountingBlockStore.cpp

@@ -1 +1,26 @@
 #include "VersionCountingBlockStore.h"
+
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+using cpputils::Data;
+using boost::none;
+namespace bf = boost::filesystem;
+
+namespace blockstore {
+    namespace versioncounting {
+
+#ifndef CRYFS_NO_COMPATIBILITY
+        void VersionCountingBlockStore::migrateFromBlockstoreWithoutVersionNumbers(BlockStore *baseBlockStore, const bf::path &integrityFilePath) {
+            std::cout << "Migrating file system for integrity features..." << std::flush;
+            KnownBlockVersions knownBlockVersions(integrityFilePath);
+            baseBlockStore->forEachBlock([&baseBlockStore, &knownBlockVersions] (const Key &key) {
+                auto block =  baseBlockStore->load(key);
+                ASSERT(block != none, "Couldn't load block for migration");
+                VersionCountingBlock::migrateFromBlockstoreWithoutVersionNumbers(std::move(*block), &knownBlockVersions);
+            });
+            std::cout << "done" << std::endl;
+        }
+#endif
+
+    }
+}

src/blockstore/implementations/versioncounting/VersionCountingBlockStore.h

@@ -25,6 +25,10 @@ public:
   uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
   void forEachBlock(std::function<void (const Key &)> callback) const override;
 
+#ifndef CRYFS_NO_COMPATIBILITY
+  static void migrateFromBlockstoreWithoutVersionNumbers(BlockStore *baseBlockStore, const boost::filesystem::path &integrityFilePath);
+#endif
+
 private:
   cpputils::unique_ref<BlockStore> _baseBlockStore;
   KnownBlockVersions _knownBlockVersions;

src/cryfs/config/CryConfig.cpp

@@ -21,11 +21,11 @@ using cpputils::Random;
 namespace cryfs {
 
 CryConfig::CryConfig()
-: _rootBlob(""), _encKey(""), _cipher(""), _version(""), _createdWithVersion(""), _blocksizeBytes(0), _filesystemId(FilesystemID::Null()) {
+: _rootBlob(""), _encKey(""), _cipher(""), _version(""), _createdWithVersion(""), _blocksizeBytes(0), _filesystemId(FilesystemID::Null()), _hasVersionNumbers(true) {
 }
 
 CryConfig::CryConfig(CryConfig &&rhs)
-: _rootBlob(std::move(rhs._rootBlob)), _encKey(std::move(rhs._encKey)), _cipher(std::move(rhs._cipher)), _version(std::move(rhs._version)), _createdWithVersion(std::move(rhs._createdWithVersion)), _blocksizeBytes(rhs._blocksizeBytes), _filesystemId(std::move(rhs._filesystemId)) {
+: _rootBlob(std::move(rhs._rootBlob)), _encKey(std::move(rhs._encKey)), _cipher(std::move(rhs._cipher)), _version(std::move(rhs._version)), _createdWithVersion(std::move(rhs._createdWithVersion)), _blocksizeBytes(rhs._blocksizeBytes), _filesystemId(std::move(rhs._filesystemId)), _hasVersionNumbers(rhs._hasVersionNumbers) {
 }
 
 CryConfig CryConfig::load(const Data &data) {
@@ -41,6 +41,9 @@ CryConfig CryConfig::load(const Data &data) {
   cfg._version = pt.get<string>("cryfs.version", "0.8"); // CryFS 0.8 didn't specify this field, so if the field doesn't exist, it's 0.8.
   cfg._createdWithVersion = pt.get<string>("cryfs.createdWithVersion", cfg._version); // In CryFS <= 0.9.2, we didn't have this field, but also didn't update cryfs.version, so we can use this field instead.
   cfg._blocksizeBytes = pt.get<uint64_t>("cryfs.blocksizeBytes", 32832); // CryFS <= 0.9.2 used a 32KB block size which was this physical block size.
+#ifndef CRYFS_NO_COMPATIBILITY
+  cfg._hasVersionNumbers = pt.get<bool>("cryfs.migrations.hasVersionNumbers", false);
+#endif
 
   optional<string> filesystemIdOpt = pt.get_optional<string>("cryfs.filesystemId");
   if (filesystemIdOpt == none) {
@@ -62,6 +65,9 @@ Data CryConfig::save() const {
   pt.put<string>("cryfs.createdWithVersion", _createdWithVersion);
   pt.put<uint64_t>("cryfs.blocksizeBytes", _blocksizeBytes);
   pt.put<string>("cryfs.filesystemId", _filesystemId.ToString());
+#ifndef CRYFS_NO_COMPATIBILITY
+  pt.put("cryfs.migrations.hasVersionNumbers", _hasVersionNumbers);
+#endif
 
   stringstream stream;
   write_json(stream, pt);
@@ -124,4 +130,14 @@ void CryConfig::SetFilesystemId(const FilesystemID &value) {
   _filesystemId = value;
 }
 
+#ifndef CRYFS_NO_COMPATIBILITY
+bool CryConfig::HasVersionNumbers() const {
+  return _hasVersionNumbers;
+}
+
+void CryConfig::SetHasVersionNumbers(bool value) {
+  _hasVersionNumbers = value;
+}
+#endif
+
 }

src/cryfs/config/CryConfig.h

@@ -38,6 +38,13 @@ public:
   const FilesystemID &FilesystemId() const;
   void SetFilesystemId(const FilesystemID &value);
 
+#ifndef CRYFS_NO_COMPATIBILITY
+  // This is a trigger to recognize old file systems that didn't have version numbers.
+  // Version numbers cannot be disabled, but the file system will be migrated to version numbers automatically.
+  bool HasVersionNumbers() const;
+  void SetHasVersionNumbers(bool value);
+#endif
+
   static CryConfig load(const cpputils::Data &data);
   cpputils::Data save() const;
 
@@ -49,6 +56,9 @@ private:
   std::string _createdWithVersion;
   uint64_t _blocksizeBytes;
   FilesystemID _filesystemId;
+#ifndef CRYFS_NO_COMPATIBILITY
+  bool _hasVersionNumbers;
+#endif
 
   DISALLOW_COPY_AND_ASSIGN(CryConfig);
 };

src/cryfs/config/CryConfigCreator.cpp

@@ -28,6 +28,9 @@ namespace cryfs {
         config.SetRootBlob(_generateRootBlobKey());
         config.SetEncryptionKey(_generateEncKey(config.Cipher()));
         config.SetFilesystemId(_generateFilesystemID());
+#ifndef CRYFS_NO_COMPATIBILITY
+        config.SetHasVersionNumbers(true);
+#endif
         return config;
     }
 

src/cryfs/config/CryConfigFile.cpp

@@ -73,6 +73,10 @@ void CryConfigFile::save() const {
 }
 
 CryConfig *CryConfigFile::config() {
+    return const_cast<CryConfig *>(const_cast<const CryConfigFile*>(this)->config());
+}
+
+const CryConfig *CryConfigFile::config() const {
     return &_config;
 }
 

src/cryfs/config/CryConfigFile.h

@@ -19,6 +19,7 @@ namespace cryfs {
         void save() const;
 
         CryConfig *config();
+        const CryConfig *config() const;
 
     private:
         CryConfigFile(const boost::filesystem::path &path, CryConfig config, cpputils::unique_ref<CryConfigEncryptor> encryptor);

src/cryfs/filesystem/CryDevice.cpp

@@ -16,6 +16,7 @@
 #include "cachingfsblobstore/CachingFsBlobStore.h"
 #include "../config/CryCipher.h"
 #include <cpp-utils/system/homedir.h>
+#include <gitversion/VersionCompare.h>
 
 using std::string;
 
@@ -30,6 +31,8 @@ using blobstore::onblocks::BlobStoreOnBlocks;
 using blobstore::onblocks::BlobOnBlocks;
 using blockstore::caching::CachingBlockStore;
 using blockstore::versioncounting::VersionCountingBlockStore;
+using blockstore::versioncounting::VersionCountingBlock;
+using gitversion::VersionCompare;
 using cpputils::unique_ref;
 using cpputils::make_unique_ref;
 using cpputils::dynamic_pointer_move;
@@ -53,14 +56,7 @@ CryDevice::CryDevice(CryConfigFile configFile, unique_ref<BlockStore> blockStore
       make_unique_ref<ParallelAccessFsBlobStore>(
         make_unique_ref<CachingFsBlobStore>(
           make_unique_ref<FsBlobStore>(
-            make_unique_ref<BlobStoreOnBlocks>(
+            CreateBlobStore(std::move(blockStore), &configFile)
-              make_unique_ref<CachingBlockStore>(
-                make_unique_ref<VersionCountingBlockStore>(
-                  CreateEncryptedBlockStore(*configFile.config(), std::move(blockStore)),
-                  _integrityFilePath(configFile.config()->FilesystemId())
-                )
-              )
-            , configFile.config()->BlocksizeBytes())
           )
         )
       )
@@ -69,6 +65,33 @@ CryDevice::CryDevice(CryConfigFile configFile, unique_ref<BlockStore> blockStore
   _onFsAction() {
 }
 
+unique_ref<blobstore::BlobStore> CryDevice::CreateBlobStore(unique_ref<BlockStore> blockStore, CryConfigFile *configFile) {
+  auto versionCountingEncryptedBlockStore = CreateVersionCountingEncryptedBlockStore(std::move(blockStore), configFile);
+  // Create versionCountingEncryptedBlockStore not in the same line as BlobStoreOnBlocks, because it can modify BlocksizeBytes
+  // in the configFile and therefore has to be run before the second parameter to the BlobStoreOnBlocks parameter is evaluated.
+  return make_unique_ref<BlobStoreOnBlocks>(
+           make_unique_ref<CachingBlockStore>(
+             std::move(versionCountingEncryptedBlockStore)
+           ),
+           configFile->config()->BlocksizeBytes());
+}
+
+unique_ref<BlockStore> CryDevice::CreateVersionCountingEncryptedBlockStore(unique_ref<BlockStore> blockStore, CryConfigFile *configFile) {
+  auto encryptedBlockStore = CreateEncryptedBlockStore(*configFile->config(), std::move(blockStore));
+  auto integrityFilePath = _integrityFilePath(configFile->config()->FilesystemId());
+
+#ifndef CRYFS_NO_COMPATIBILITY
+  if (!configFile->config()->HasVersionNumbers()) {
+    VersionCountingBlockStore::migrateFromBlockstoreWithoutVersionNumbers(encryptedBlockStore.get(), integrityFilePath);
+    configFile->config()->SetBlocksizeBytes(configFile->config()->BlocksizeBytes() + VersionCountingBlock::HEADER_LENGTH);
+    configFile->config()->SetHasVersionNumbers(true);
+    configFile->save();
+  }
+#endif
+
+  return make_unique_ref<VersionCountingBlockStore>(std::move(encryptedBlockStore), integrityFilePath);
+}
+
 Key CryDevice::CreateRootBlobAndReturnKey() {
   auto rootBlob =  _fsBlobStore->createDirBlob();
   rootBlob->flush(); // Don't cache, but directly write the root blob (this causes it to fail early if the base directory is not accessible)

src/cryfs/filesystem/CryDevice.h

@@ -52,6 +52,8 @@ private:
 
   blockstore::Key GetOrCreateRootKey(CryConfigFile *config);
   blockstore::Key CreateRootBlobAndReturnKey();
+  static cpputils::unique_ref<blobstore::BlobStore> CreateBlobStore(cpputils::unique_ref<blockstore::BlockStore> blockStore, CryConfigFile *configFile);
+  static cpputils::unique_ref<blockstore::BlockStore> CreateVersionCountingEncryptedBlockStore(cpputils::unique_ref<blockstore::BlockStore> blockStore, CryConfigFile *configFile);
   static cpputils::unique_ref<blockstore::BlockStore> CreateEncryptedBlockStore(const CryConfig &config, cpputils::unique_ref<blockstore::BlockStore> baseBlockStore);
 
   struct BlobWithParent {