From 9c6713a00e47544582db2493aad012633cafbb22 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sat, 10 Nov 2018 12:38:57 -0800 Subject: [PATCH] Fix cryfs-stat --- .../onblocks/datanodestore/DataInnerNode.cpp | 2 +- .../onblocks/datanodestore/DataNodeStore.cpp | 4 + .../onblocks/datanodestore/DataNodeStore.h | 2 + src/cryfs/config/CryConfigLoader.cpp | 4 + src/cryfs/config/CryConfigLoader.h | 1 + src/stats/main.cpp | 189 +++++++++++------- 6 files changed, 124 insertions(+), 78 deletions(-) diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp b/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp index 87c7f3fd..d64c0d4a 100644 --- a/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp +++ b/src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp @@ -18,7 +18,7 @@ DataInnerNode::DataInnerNode(DataNodeView view) : DataNode(std::move(view)) { ASSERT(depth() > 0, "Inner node can't have depth 0. Is this a leaf maybe?"); if (node().FormatVersion() != FORMAT_VERSION_HEADER) { - throw std::runtime_error("This node format is not supported. Was it created with a newer version of CryFS?"); + throw std::runtime_error("This node format (" + std::to_string(node().FormatVersion()) + ") is not supported. Was it created with a newer version of CryFS?"); } } diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp index 8ccc180b..5149b2e3 100644 --- a/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp +++ b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp @@ -137,6 +137,10 @@ DataNodeLayout DataNodeStore::layout() const { return _layout; } +void DataNodeStore::forEachNode(std::function callback) const { + _blockstore->forEachBlock(std::move(callback)); +} + } } } diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h index 8bb8c8ad..74709ed9 100644 --- a/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h +++ b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h @@ -51,6 +51,8 @@ public: uint64_t estimateSpaceForNumNodesLeft() const; //TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree() + void forEachNode(std::function callback) const; + private: cpputils::unique_ref _blockstore; diff --git a/src/cryfs/config/CryConfigLoader.cpp b/src/cryfs/config/CryConfigLoader.cpp index 399bc9a9..84087709 100644 --- a/src/cryfs/config/CryConfigLoader.cpp +++ b/src/cryfs/config/CryConfigLoader.cpp @@ -102,6 +102,10 @@ void CryConfigLoader::_checkMissingBlocksAreIntegrityViolations(CryConfigFile *c } } +optional CryConfigLoader::load(bf::path filename, bool allowFilesystemUpgrade, bool allowReplacedFilesystem) { + return _loadConfig(std::move(filename), allowFilesystemUpgrade, allowReplacedFilesystem); +} + optional CryConfigLoader::loadOrCreate(bf::path filename, bool allowFilesystemUpgrade, bool allowReplacedFilesystem) { if (bf::exists(filename)) { return _loadConfig(std::move(filename), allowFilesystemUpgrade, allowReplacedFilesystem); diff --git a/src/cryfs/config/CryConfigLoader.h b/src/cryfs/config/CryConfigLoader.h index 25c4b6ab..80f4bcde 100644 --- a/src/cryfs/config/CryConfigLoader.h +++ b/src/cryfs/config/CryConfigLoader.h @@ -23,6 +23,7 @@ public: }; boost::optional loadOrCreate(boost::filesystem::path filename, bool allowFilesystemUpgrade, bool allowReplacedFilesystem); + boost::optional load(boost::filesystem::path filename, bool allowFilesystemUpgrade, bool allowReplacedFilesystem); private: boost::optional _loadConfig(boost::filesystem::path filename, bool allowFilesystemUpgrade, bool allowReplacedFilesystem); diff --git a/src/stats/main.cpp b/src/stats/main.cpp index 1ecfe080..3d59a13c 100644 --- a/src/stats/main.cpp +++ b/src/stats/main.cpp @@ -1,8 +1,9 @@ #include #include -#include +#include #include #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include @@ -23,13 +25,14 @@ using namespace cryfs; using namespace cpputils; using namespace blockstore; using namespace blockstore::ondisk; +using namespace blockstore::integrity; using namespace blockstore::lowtohighlevel; using namespace blobstore::onblocks; using namespace blobstore::onblocks::datanodestore; using namespace cryfs::fsblobstore; void printNode(unique_ref node) { - std::cout << "BlockId: " << node->blockId().ToString() << ", Depth: " << node->depth() << " "; + std::cout << "BlockId: " << node->blockId().ToString() << ", Depth: " << static_cast(node->depth()) << " "; auto innerNode = dynamic_pointer_move(node); if (innerNode != none) { std::cout << "Type: inner\n"; @@ -42,111 +45,143 @@ void printNode(unique_ref node) { } } -set _getBlockstoreUnaccountedBlocks(const CryConfig &config) { - auto onDiskBlockStore = make_unique_ref("/home/heinzi/basedir"); - auto encryptedBlockStore = CryCiphers::find(config.Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config.EncryptionKey()); - auto highLevelBlockStore = make_unique_ref(std::move(encryptedBlockStore)); - auto nodeStore = make_unique_ref(std::move(highLevelBlockStore), config.BlocksizeBytes()); - std::set unaccountedBlocks; - uint32_t numBlocks = nodeStore->numNodes(); - uint32_t i = 0; - cout << "There are " << nodeStore->numNodes() << " blocks." << std::endl; - // Add all blocks to unaccountedBlocks - for (auto file = directory_iterator("/home/heinzi/basedir"); file != directory_iterator(); ++file) { - cout << "\r" << (++i) << "/" << numBlocks << flush; - if (file->path().filename() != "cryfs.config") { - auto blockId = BlockId::FromString(file->path().filename().string().c_str()); - unaccountedBlocks.insert(blockId); +void _forEachBlob(FsBlobStore* blobStore, const BlockId& rootId, std::function callback) { + callback(rootId); + auto rootBlob = blobStore->load(rootId); + ASSERT(rootBlob != boost::none, "Blob not found but referenced from directory entry"); + + auto rootDir = dynamic_pointer_move(*rootBlob); + if (rootDir != boost::none) { + std::vector children; + children.reserve((*rootDir)->NumChildren()); + (*rootDir)->AppendChildrenTo(&children); + + for (const auto& child : children) { + auto childEntry = (*rootDir)->GetChild(child.name); + ASSERT(childEntry != boost::none, "We just got this from the entry list, it must exist."); + auto childId = childEntry->blockId(); + _forEachBlob(blobStore, childId, callback); } } - i = 0; - cout << "\nRemove blocks that have a parent" << endl; - //Remove root block from unaccountedBlocks - unaccountedBlocks.erase(BlockId::FromString(config.RootBlob())); - //Remove all blocks that have a parent node from unaccountedBlocks - for (auto file = directory_iterator("/home/heinzi/basedir"); file != directory_iterator(); ++file) { - cout << "\r" << (++i) << "/" << numBlocks << flush; - if (file->path().filename() != "cryfs.config") { - auto blockId = BlockId::FromString(file->path().filename().string().c_str()); - auto node = nodeStore->load(blockId); - auto innerNode = dynamic_pointer_move(*node); - if (innerNode != none) { - for (uint32_t childIndex = 0; childIndex < (*innerNode)->numChildren(); ++childIndex) { - auto child = (*innerNode)->readChild(childIndex).blockId(); - unaccountedBlocks.erase(child); - } - } - } - } - return unaccountedBlocks; } -set _getBlocksReferencedByDirEntries(const CryConfig &config) { - auto onDiskBlockStore = make_unique_ref("/home/heinzi/basedir"); - auto encryptedBlockStore = CryCiphers::find(config.Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config.EncryptionKey()); - auto highLevelBlockStore = make_unique_ref(std::move(encryptedBlockStore)); - auto fsBlobStore = make_unique_ref(make_unique_ref(std::move(highLevelBlockStore), config.BlocksizeBytes())); - set blocksReferencedByDirEntries; - uint32_t numBlocks = fsBlobStore->numBlocks(); - uint32_t i = 0; - cout << "\nRemove blocks referenced by dir entries" << endl; - for (auto file = directory_iterator("/home/heinzi/basedir"); file != directory_iterator(); ++file) { - cout << "\r" << (++i) << "/" << numBlocks << flush; - if (file->path().filename() != "cryfs.config") { - auto blockId = BlockId::FromString(file->path().filename().string().c_str()); - try { - auto blob = fsBlobStore->load(blockId); - if (blob != none) { - auto dir = dynamic_pointer_move(*blob); - if (dir != none) { - vector children; - (*dir)->AppendChildrenTo(&children); - for (const auto &child : children) { - blocksReferencedByDirEntries.insert((*dir)->GetChild(child.name)->blockId()); - } - } - } - } catch (...) {} +void _forEachBlockInBlob(DataNodeStore* nodeStore, const BlockId& rootId, std::function callback) { + callback(rootId); + + auto node = nodeStore->load(rootId); + auto innerNode = dynamic_pointer_move(*node); + if (innerNode != boost::none) { + for (uint32_t childIndex = 0; childIndex < (*innerNode)->numChildren(); ++childIndex) { + auto childId = (*innerNode)->readChild(childIndex).blockId(); + _forEachBlockInBlob(nodeStore, childId, callback); } } - return blocksReferencedByDirEntries; +} + +unique_ref makeBlockStore(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) { + auto onDiskBlockStore = make_unique_ref(basedir); + auto encryptedBlockStore = CryCiphers::find(config.configFile.config()->Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config.configFile.config()->EncryptionKey()); + auto statePath = localStateDir.forFilesystemId(config.configFile.config()->FilesystemId()); + auto integrityFilePath = statePath / "integritydata"; + auto integrityBlockStore = make_unique_ref(std::move(encryptedBlockStore), integrityFilePath, config.myClientId, false, true); + return make_unique_ref(std::move(integrityBlockStore)); +} + +std::vector _getKnownBlobIds(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) { + auto blockStore = makeBlockStore(basedir, config, localStateDir); + auto fsBlobStore = make_unique_ref(make_unique_ref(std::move(blockStore), config.configFile.config()->BlocksizeBytes())); + + std::vector result; + cout << "Listing all file system entities (i.e. blobs)..." << flush; + auto rootId = BlockId::FromString(config.configFile.config()->RootBlob()); + _forEachBlob(fsBlobStore.get(), rootId, [&result] (const BlockId& blockId) { + result.push_back(blockId); + }); + cout << "done" << endl; + return result; +} + +std::vector _getKnownBlockIds(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) { + auto knownBlobIds = _getKnownBlobIds(basedir, config, localStateDir); + + auto blockStore = makeBlockStore(basedir, config, localStateDir); + auto nodeStore = make_unique_ref(std::move(blockStore), config.configFile.config()->BlocksizeBytes()); + std::vector result; + const uint32_t numNodes = nodeStore->numNodes(); + result.reserve(numNodes); + uint32_t i = 0; + cout << "Listing all blocks used by these file system entities..." << endl; + for (const auto& blobId : knownBlobIds) { + _forEachBlockInBlob(nodeStore.get(), blobId, [&result, &i, numNodes] (const BlockId& blockId) { + cout << "\r" << (++i) << "/" << numNodes << flush; + result.push_back(blockId); + }); + } + std::cout << "...done" << endl; + return result; +} + +set _getAllBlockIds(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) { + auto blockStore= makeBlockStore(basedir, config, localStateDir); + set result; + blockStore->forEachBlock([&result] (const BlockId& blockId) { + result.insert(blockId); + }); + return result; } -int main() { +int main(int argc, char* argv[]) { + if (argc != 2) { + std::cerr << "Usage: cryfs-stats [basedir]" << std::endl; + exit(1); + } + path basedir = argv[1]; + std::cout << "Calculating stats for filesystem at " << basedir << std::endl; + auto console = std::make_shared(); console->print("Loading config\n"); auto askPassword = [console] () { return console->askPassword("Password: "); }; - auto keyProvider = make_unique_ref( + unique_ref keyProvider = make_unique_ref( console, askPassword, askPassword, make_unique_ref(SCrypt::DefaultSettings) ); - auto config = CryConfigFile::load("/home/heinzi/basedir/cryfs.config", keyProvider.get()); - set unaccountedBlocks = _getBlockstoreUnaccountedBlocks(*config->config()); - //Remove all blocks that are referenced by a directory entry from unaccountedBlocks - set blocksReferencedByDirEntries = _getBlocksReferencedByDirEntries(*config->config()); - for (const auto &blockId : blocksReferencedByDirEntries) { - unaccountedBlocks.erase(blockId); + auto config_path = basedir / "cryfs.config"; + LocalStateDir localStateDir(cpputils::system::HomeDirectory::getXDGDataDir() / "cryfs"); + CryConfigLoader config_loader(console, Random::OSRandom(), std::move(keyProvider), localStateDir, boost::none, boost::none, boost::none); + + auto config = config_loader.load(config_path, false, true); + if (config == boost::none) { + std::cerr << "Error loading config file" << std::endl; + exit(1); + } + + cout << "Listing all blocks..." << flush; + set unaccountedBlocks = _getAllBlockIds(basedir, *config, localStateDir); + cout << "done" << endl; + + vector accountedBlocks = _getKnownBlockIds(basedir, *config, localStateDir); + for (const BlockId& blockId : accountedBlocks) { + auto num_erased = unaccountedBlocks.erase(blockId); + ASSERT(1 == num_erased, "Blob id referenced by directory entry but didn't found it on disk? This can't happen."); } console->print("Calculate statistics\n"); - auto onDiskBlockStore = make_unique_ref("/home/heinzi/basedir"); - auto encryptedBlockStore = CryCiphers::find(config->config()->Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config->config()->EncryptionKey()); - auto highLevelBlockStore = make_unique_ref(std::move(encryptedBlockStore)); - auto nodeStore = make_unique_ref(std::move(highLevelBlockStore), config->config()->BlocksizeBytes()); + auto blockStore = makeBlockStore(basedir, *config, localStateDir); + auto nodeStore = make_unique_ref(std::move(blockStore), config->configFile.config()->BlocksizeBytes()); uint32_t numUnaccountedBlocks = unaccountedBlocks.size(); uint32_t numLeaves = 0; uint32_t numInner = 0; console->print("Unaccounted blocks: " + std::to_string(unaccountedBlocks.size()) + "\n"); for (const auto &blockId : unaccountedBlocks) { - console->print("\r" + std::to_string(numLeaves+numInner) + "/" + std::to_string(numUnaccountedBlocks)); + console->print("\r" + std::to_string(numLeaves+numInner) + "/" + std::to_string(numUnaccountedBlocks) + ": "); auto node = nodeStore->load(blockId); auto innerNode = dynamic_pointer_move(*node); if (innerNode != none) {