Refactor traversl in cryfs-stats

src/stats/CMakeLists.txt

@@ -2,6 +2,7 @@ project (stats)
 
 set(SOURCES
         main.cpp
+        traversal.cpp
 )
 
 add_executable(${PROJECT_NAME} ${SOURCES})

src/stats/main.cpp

@@ -15,6 +15,7 @@
 #include <cryfs/impl/filesystem/CryDevice.h>
 #include <cpp-utils/io/IOStreamConsole.h>
 #include <cpp-utils/system/homedir.h>
+#include "traversal.h"
 
 #include <set>
 
@@ -31,6 +32,8 @@ using namespace blobstore::onblocks;
 using namespace blobstore::onblocks::datanodestore;
 using namespace cryfs::fsblobstore;
 
+using namespace cryfs_stats;
+
 void printNode(unique_ref<DataNode> node) {
     std::cout << "BlockId: " << node->blockId().ToString() << ", Depth: " << static_cast<int>(node->depth()) << " ";
     auto innerNode = dynamic_pointer_move<DataInnerNode>(node);
@@ -45,39 +48,6 @@ void printNode(unique_ref<DataNode> node) {
     }
 }
 
-void _forEachBlob(FsBlobStore* blobStore, const BlockId& rootId, std::function<void (const BlockId& blobId)> callback) {
-    callback(rootId);
-    auto rootBlob = blobStore->load(rootId);
-    ASSERT(rootBlob != boost::none, "Blob not found but referenced from directory entry");
-
-    auto rootDir = dynamic_pointer_move<DirBlob>(*rootBlob);
-    if (rootDir != boost::none) {
-        std::vector<fspp::Dir::Entry> children;
-        children.reserve((*rootDir)->NumChildren());
-        (*rootDir)->AppendChildrenTo(&children);
-
-        for (const auto& child : children) {
-            auto childEntry = (*rootDir)->GetChild(child.name);
-            ASSERT(childEntry != boost::none, "We just got this from the entry list, it must exist.");
-            auto childId = childEntry->blockId();
-            _forEachBlob(blobStore, childId, callback);
-        }
-    }
-}
-
-void _forEachBlockInBlob(DataNodeStore* nodeStore, const BlockId& rootId, std::function<void (const BlockId& blockId)> callback) {
-    callback(rootId);
-
-    auto node = nodeStore->load(rootId);
-    auto innerNode = dynamic_pointer_move<DataInnerNode>(*node);
-    if (innerNode != boost::none) {
-        for (uint32_t childIndex = 0; childIndex < (*innerNode)->numChildren(); ++childIndex) {
-            auto childId = (*innerNode)->readChild(childIndex).blockId();
-            _forEachBlockInBlob(nodeStore, childId, callback);
-        }
-    }
-}
-
 unique_ref<BlockStore> makeBlockStore(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) {
     auto onDiskBlockStore = make_unique_ref<OnDiskBlockStore2>(basedir);
     auto encryptedBlockStore = CryCiphers::find(config.configFile->config()->Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config.configFile->config()->EncryptionKey());
@@ -90,18 +60,52 @@ unique_ref<BlockStore> makeBlockStore(const path& basedir, const CryConfigLoader
     return make_unique_ref<LowToHighLevelBlockStore>(std::move(integrityBlockStore));
 }
 
+struct AccumulateBlockIds final {
+public:
+    auto callback() {
+        return [this] (const BlockId& id) {
+            _blockIds.push_back(id);
+        };
+    }
+
+    const std::vector<BlockId>& blockIds() const {
+        return _blockIds;
+    }
+
+    void reserve(size_t size) {
+        _blockIds.reserve(size);
+    }
+
+private:
+    std::vector<BlockId> _blockIds;
+};
+
+class ProgressBar final {
+public:
+    ProgressBar(size_t numBlocks): _currentBlock(0), _numBlocks(numBlocks) {}
+
+    auto callback() {
+        return [this] (const BlockId&) {
+            cout << "\r" << (++_currentBlock) << "/" << _numBlocks << flush;
+        };
+    }
+private:
+    size_t _currentBlock;
+    size_t _numBlocks;
+};
+
 std::vector<BlockId> _getKnownBlobIds(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) {
     auto blockStore = makeBlockStore(basedir, config, localStateDir);
     auto fsBlobStore = make_unique_ref<FsBlobStore>(make_unique_ref<BlobStoreOnBlocks>(std::move(blockStore), config.configFile->config()->BlocksizeBytes()));
 
     std::vector<BlockId> result;
+    AccumulateBlockIds knownBlobIds;
     cout << "Listing all file system entities (i.e. blobs)..." << flush;
     auto rootId = BlockId::FromString(config.configFile->config()->RootBlob());
-    _forEachBlob(fsBlobStore.get(), rootId, [&result] (const BlockId& blockId) {
+    forEachReachableBlob(fsBlobStore.get(), rootId, {knownBlobIds.callback()});
-        result.push_back(blockId);
-    });
     cout << "done" << endl;
-    return result;
+
+    return knownBlobIds.blockIds();
 }
 
 std::vector<BlockId> _getKnownBlockIds(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) {
@@ -109,31 +113,29 @@ std::vector<BlockId> _getKnownBlockIds(const path& basedir, const CryConfigLoade
 
     auto blockStore = makeBlockStore(basedir, config, localStateDir);
     auto nodeStore = make_unique_ref<DataNodeStore>(std::move(blockStore), config.configFile->config()->BlocksizeBytes());
-    std::vector<BlockId> result;
+    AccumulateBlockIds knownBlockIds;
     const uint32_t numNodes = nodeStore->numNodes();
-    result.reserve(numNodes);
+    knownBlockIds.reserve(numNodes);
     uint32_t i = 0;
     cout << "Listing all blocks used by these file system entities..." << endl;
     for (const auto& blobId : knownBlobIds) {
-        _forEachBlockInBlob(nodeStore.get(), blobId, [&result, &i, numNodes] (const BlockId& blockId) {
+        forEachReachableBlockInBlob(nodeStore.get(), blobId, {
-            cout << "\r" << (++i) << "/" << numNodes << flush;
+            ProgressBar(numNodes).callback(),
-            result.push_back(blockId);
+            knownBlockIds.callback()
         });
     }
     std::cout << "...done" << endl;
-    return result;
+    return knownBlockIds.blockIds();
 }
 
 set<BlockId> _getAllBlockIds(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) {
-    auto blockStore= makeBlockStore(basedir, config, localStateDir);
+    auto blockStore = makeBlockStore(basedir, config, localStateDir);
-    set<BlockId> result;
+    AccumulateBlockIds allBlockIds;
-    blockStore->forEachBlock([&result] (const BlockId& blockId) {
+    allBlockIds.reserve(blockStore->numBlocks());
-        result.insert(blockId);
+    forEachBlock(blockStore.get(), {allBlockIds.callback()});
-    });
+    return set<BlockId>(allBlockIds.blockIds().begin(), allBlockIds.blockIds().end());
-    return result;
 }
 
-
 int main(int argc, char* argv[]) {
     if (argc != 2) {
         std::cerr << "Usage: cryfs-stats [basedir]" << std::endl;
@@ -167,7 +169,7 @@ int main(int argc, char* argv[]) {
     const auto& config_ = config->configFile->config();
     std::cout << "Loading filesystem of version " << config_->Version() << std::endl;
 #ifndef CRYFS_NO_COMPATIBILITY
-    const bool is_correct_format = config_->Version() == CryConfig::FilesystemFormatVersion && !config_->HasParentPointers() && !config_->HasVersionNumbers();
+    const bool is_correct_format = config_->Version() == CryConfig::FilesystemFormatVersion && config_->HasParentPointers() && config_->HasVersionNumbers();
 #else
     const bool is_correct_format = config_->Version() == CryConfig::FilesystemFormatVersion;
 #endif

src/stats/traversal.cpp

@@ -0,0 +1,65 @@
+#include "traversal.h"
+
+#include <blobstore/implementations/onblocks/datanodestore/DataInnerNode.h>
+
+using blockstore::BlockId;
+using blockstore::BlockStore;
+using cryfs::fsblobstore::FsBlobStore;
+using cryfs::fsblobstore::DirBlob;
+using blobstore::onblocks::datanodestore::DataNodeStore;
+using blobstore::onblocks::datanodestore::DataInnerNode;
+using cpputils::dynamic_pointer_move;
+
+using std::vector;
+using std::function;
+using boost::none;
+
+namespace cryfs_stats {
+
+void forEachBlock(BlockStore* blockStore, const vector<function<void (const BlockId& blobId)>>& callbacks) {
+    blockStore->forEachBlock([&callbacks] (const BlockId& blockId) {
+        for(const auto& callback : callbacks) {
+            callback(blockId);
+        }
+    });
+}
+
+void forEachReachableBlob(FsBlobStore* blobStore, const BlockId& rootId, const vector<function<void (const BlockId& blobId)>>& callbacks) {
+    for (const auto& callback : callbacks) {
+        callback(rootId);
+    }
+
+    auto rootBlob = blobStore->load(rootId);
+    ASSERT(rootBlob != none, "Blob not found but referenced from directory entry");
+
+    auto rootDir = dynamic_pointer_move<DirBlob>(*rootBlob);
+    if (rootDir != none) {
+        vector<fspp::Dir::Entry> children;
+        children.reserve((*rootDir)->NumChildren());
+        (*rootDir)->AppendChildrenTo(&children);
+
+        for (const auto& child : children) {
+            auto childEntry = (*rootDir)->GetChild(child.name);
+            ASSERT(childEntry != none, "We just got this from the entry list, it must exist.");
+            auto childId = childEntry->blockId();
+            forEachReachableBlob(blobStore, childId, callbacks);
+        }
+    }
+}
+
+void forEachReachableBlockInBlob(DataNodeStore* nodeStore, const BlockId& rootId, const vector<function<void (const BlockId& blockId)>>& callbacks) {
+    for (const auto& callback : callbacks) {
+        callback(rootId);
+    }
+
+    auto node = nodeStore->load(rootId);
+    auto innerNode = dynamic_pointer_move<DataInnerNode>(*node);
+    if (innerNode != none) {
+        for (uint32_t childIndex = 0; childIndex < (*innerNode)->numChildren(); ++childIndex) {
+            auto childId = (*innerNode)->readChild(childIndex).blockId();
+            forEachReachableBlockInBlob(nodeStore, childId, callbacks);
+        }
+    }
+}
+
+}

src/stats/traversal.h

@@ -0,0 +1,24 @@
+#pragma once
+#ifndef CRYFS_STATS_TRAVERSAL_H
+#define CRYFS_STATS_TRAVERSAL_H
+
+#include <vector>
+#include <functional>
+#include <blockstore/interface/Block.h>
+#include <blobstore/implementations/onblocks/datanodestore/DataNodeStore.h>
+#include <cryfs/impl/filesystem/fsblobstore/FsBlobStore.h>
+
+namespace cryfs_stats {
+
+    // Call the callbacks on each existing block, whether it is connected or orphaned
+    void forEachBlock(blockstore::BlockStore* blockStore, const std::vector<std::function<void (const blockstore::BlockId& blobId)>>& callbacks);
+
+    // Call the callbacks on each existing blob that is reachable from the root blob, i.e. not orphaned
+    void forEachReachableBlob(cryfs::fsblobstore::FsBlobStore* blobStore, const blockstore::BlockId& rootId, const std::vector<std::function<void (const blockstore::BlockId& blobId)>>& callbacks);
+
+    // Call the callbacks on each block that is reachable from the given blob root, i.e. belongs to this blob.
+    void forEachReachableBlockInBlob(blobstore::onblocks::datanodestore::DataNodeStore* nodeStore, const blockstore::BlockId& rootId, const std::vector<std::function<void (const blockstore::BlockId& blockId)>>& callbacks);
+
+}
+
+#endif