hardcoresushi/libcryfs
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Fix cryfs-stat

Browse Source
This commit is contained in:
Sebastian Messmer 2018-11-10 12:38:57 -08:00
parent d761dba894
commit 9c6713a00e
6 changed files with 124 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp
View File

@ -18,7 +18,7 @@ DataInnerNode::DataInnerNode(DataNodeView view)
: DataNode(std::move(view)) {
ASSERT(depth() > 0, "Inner node can't have depth 0. Is this a leaf maybe?");
if (node().FormatVersion() != FORMAT_VERSION_HEADER) {
throw std::runtime_error("This node format is not supported. Was it created with a newer version of CryFS?");
throw std::runtime_error("This node format (" + std::to_string(node().FormatVersion()) + ") is not supported. Was it created with a newer version of CryFS?");
}
}

4
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp
View File

@ -137,6 +137,10 @@ DataNodeLayout DataNodeStore::layout() const {
return _layout;
}
void DataNodeStore::forEachNode(std::function<void (const BlockId& nodeId)> callback) const {
_blockstore->forEachBlock(std::move(callback));
}
}
}
}

2
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h
View File

@ -51,6 +51,8 @@ public:
uint64_t estimateSpaceForNumNodesLeft() const;
//TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree()
void forEachNode(std::function<void (const blockstore::BlockId& nodeId)> callback) const;
private:
cpputils::unique_ref<blockstore::BlockStore> _blockstore;

4
src/cryfs/config/CryConfigLoader.cpp
View File

@ -102,6 +102,10 @@ void CryConfigLoader::_checkMissingBlocksAreIntegrityViolations(CryConfigFile *c
}
}
optional<CryConfigLoader::ConfigLoadResult> CryConfigLoader::load(bf::path filename, bool allowFilesystemUpgrade, bool allowReplacedFilesystem) {
return _loadConfig(std::move(filename), allowFilesystemUpgrade, allowReplacedFilesystem);
}
optional<CryConfigLoader::ConfigLoadResult> CryConfigLoader::loadOrCreate(bf::path filename, bool allowFilesystemUpgrade, bool allowReplacedFilesystem) {
if (bf::exists(filename)) {
return _loadConfig(std::move(filename), allowFilesystemUpgrade, allowReplacedFilesystem);

1
src/cryfs/config/CryConfigLoader.h
View File

@ -23,6 +23,7 @@ public:
};
boost::optional<ConfigLoadResult> loadOrCreate(boost::filesystem::path filename, bool allowFilesystemUpgrade, bool allowReplacedFilesystem);
boost::optional<ConfigLoadResult> load(boost::filesystem::path filename, bool allowFilesystemUpgrade, bool allowReplacedFilesystem);
private:
boost::optional<ConfigLoadResult> _loadConfig(boost::filesystem::path filename, bool allowFilesystemUpgrade, bool allowReplacedFilesystem);

189
src/stats/main.cpp
View File

@ -1,8 +1,9 @@
#include <iostream>
#include <boost/filesystem.hpp>
#include <cryfs/config/CryConfigFile.h>
#include <cryfs/config/CryConfigLoader.h>
#include <cryfs/config/CryPasswordBasedKeyProvider.h>
#include <blockstore/implementations/ondisk/OnDiskBlockStore2.h>
#include <blockstore/implementations/integrity/IntegrityBlockStore2.h>
#include <blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h>
#include <blobstore/implementations/onblocks/datanodestore/DataNodeStore.h>
#include <blobstore/implementations/onblocks/datanodestore/DataNode.h>
@ -13,6 +14,7 @@
#include <cryfs/filesystem/fsblobstore/DirBlob.h>
#include <cryfs/filesystem/CryDevice.h>
#include <cpp-utils/io/IOStreamConsole.h>
#include <cpp-utils/system/homedir.h>
#include <set>
@ -23,13 +25,14 @@ using namespace cryfs;
using namespace cpputils;
using namespace blockstore;
using namespace blockstore::ondisk;
using namespace blockstore::integrity;
using namespace blockstore::lowtohighlevel;
using namespace blobstore::onblocks;
using namespace blobstore::onblocks::datanodestore;
using namespace cryfs::fsblobstore;
void printNode(unique_ref<DataNode> node) {
std::cout << "BlockId: " << node->blockId().ToString() << ", Depth: " << node->depth() << " ";
std::cout << "BlockId: " << node->blockId().ToString() << ", Depth: " << static_cast<int>(node->depth()) << " ";
auto innerNode = dynamic_pointer_move<DataInnerNode>(node);
if (innerNode != none) {
std::cout << "Type: inner\n";
@ -42,111 +45,143 @@ void printNode(unique_ref<DataNode> node) {
}
}
set<BlockId> _getBlockstoreUnaccountedBlocks(const CryConfig &config) {
auto onDiskBlockStore = make_unique_ref<OnDiskBlockStore2>("/home/heinzi/basedir");
auto encryptedBlockStore = CryCiphers::find(config.Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config.EncryptionKey());
auto highLevelBlockStore = make_unique_ref<LowToHighLevelBlockStore>(std::move(encryptedBlockStore));
auto nodeStore = make_unique_ref<DataNodeStore>(std::move(highLevelBlockStore), config.BlocksizeBytes());
std::set<BlockId> unaccountedBlocks;
uint32_t numBlocks = nodeStore->numNodes();
uint32_t i = 0;
cout << "There are " << nodeStore->numNodes() << " blocks." << std::endl;
// Add all blocks to unaccountedBlocks
for (auto file = directory_iterator("/home/heinzi/basedir"); file != directory_iterator(); ++file) {
cout << "\r" << (++i) << "/" << numBlocks << flush;
if (file->path().filename() != "cryfs.config") {
auto blockId = BlockId::FromString(file->path().filename().string().c_str());
unaccountedBlocks.insert(blockId);
void _forEachBlob(FsBlobStore* blobStore, const BlockId& rootId, std::function<void (const BlockId& blobId)> callback) {
callback(rootId);
auto rootBlob = blobStore->load(rootId);
ASSERT(rootBlob != boost::none, "Blob not found but referenced from directory entry");
auto rootDir = dynamic_pointer_move<DirBlob>(*rootBlob);
if (rootDir != boost::none) {
std::vector<fspp::Dir::Entry> children;
children.reserve((*rootDir)->NumChildren());
(*rootDir)->AppendChildrenTo(&children);
for (const auto& child : children) {
auto childEntry = (*rootDir)->GetChild(child.name);
ASSERT(childEntry != boost::none, "We just got this from the entry list, it must exist.");
auto childId = childEntry->blockId();
_forEachBlob(blobStore, childId, callback);
}
}
i = 0;
cout << "\nRemove blocks that have a parent" << endl;
//Remove root block from unaccountedBlocks
unaccountedBlocks.erase(BlockId::FromString(config.RootBlob()));
//Remove all blocks that have a parent node from unaccountedBlocks
for (auto file = directory_iterator("/home/heinzi/basedir"); file != directory_iterator(); ++file) {
cout << "\r" << (++i) << "/" << numBlocks << flush;
if (file->path().filename() != "cryfs.config") {
auto blockId = BlockId::FromString(file->path().filename().string().c_str());
auto node = nodeStore->load(blockId);
auto innerNode = dynamic_pointer_move<DataInnerNode>(*node);
if (innerNode != none) {
for (uint32_t childIndex = 0; childIndex < (*innerNode)->numChildren(); ++childIndex) {
auto child = (*innerNode)->readChild(childIndex).blockId();
unaccountedBlocks.erase(child);
}
}
}
}
return unaccountedBlocks;
}
set<BlockId> _getBlocksReferencedByDirEntries(const CryConfig &config) {
auto onDiskBlockStore = make_unique_ref<OnDiskBlockStore2>("/home/heinzi/basedir");
auto encryptedBlockStore = CryCiphers::find(config.Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config.EncryptionKey());
auto highLevelBlockStore = make_unique_ref<LowToHighLevelBlockStore>(std::move(encryptedBlockStore));
auto fsBlobStore = make_unique_ref<FsBlobStore>(make_unique_ref<BlobStoreOnBlocks>(std::move(highLevelBlockStore), config.BlocksizeBytes()));
set<BlockId> blocksReferencedByDirEntries;
uint32_t numBlocks = fsBlobStore->numBlocks();
uint32_t i = 0;
cout << "\nRemove blocks referenced by dir entries" << endl;
for (auto file = directory_iterator("/home/heinzi/basedir"); file != directory_iterator(); ++file) {
cout << "\r" << (++i) << "/" << numBlocks << flush;
if (file->path().filename() != "cryfs.config") {
auto blockId = BlockId::FromString(file->path().filename().string().c_str());
try {
auto blob = fsBlobStore->load(blockId);
if (blob != none) {
auto dir = dynamic_pointer_move<DirBlob>(*blob);
if (dir != none) {
vector<fspp::Dir::Entry> children;
(*dir)->AppendChildrenTo(&children);
for (const auto &child : children) {
blocksReferencedByDirEntries.insert((*dir)->GetChild(child.name)->blockId());
}
}
}
} catch (...) {}
void _forEachBlockInBlob(DataNodeStore* nodeStore, const BlockId& rootId, std::function<void (const BlockId& blockId)> callback) {
callback(rootId);
auto node = nodeStore->load(rootId);
auto innerNode = dynamic_pointer_move<DataInnerNode>(*node);
if (innerNode != boost::none) {
for (uint32_t childIndex = 0; childIndex < (*innerNode)->numChildren(); ++childIndex) {
auto childId = (*innerNode)->readChild(childIndex).blockId();
_forEachBlockInBlob(nodeStore, childId, callback);
}
}
return blocksReferencedByDirEntries;
}
unique_ref<BlockStore> makeBlockStore(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) {
auto onDiskBlockStore = make_unique_ref<OnDiskBlockStore2>(basedir);
auto encryptedBlockStore = CryCiphers::find(config.configFile.config()->Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config.configFile.config()->EncryptionKey());
auto statePath = localStateDir.forFilesystemId(config.configFile.config()->FilesystemId());
auto integrityFilePath = statePath / "integritydata";
auto integrityBlockStore = make_unique_ref<IntegrityBlockStore2>(std::move(encryptedBlockStore), integrityFilePath, config.myClientId, false, true);
return make_unique_ref<LowToHighLevelBlockStore>(std::move(integrityBlockStore));
}
std::vector<BlockId> _getKnownBlobIds(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) {
auto blockStore = makeBlockStore(basedir, config, localStateDir);
auto fsBlobStore = make_unique_ref<FsBlobStore>(make_unique_ref<BlobStoreOnBlocks>(std::move(blockStore), config.configFile.config()->BlocksizeBytes()));
std::vector<BlockId> result;
cout << "Listing all file system entities (i.e. blobs)..." << flush;
auto rootId = BlockId::FromString(config.configFile.config()->RootBlob());
_forEachBlob(fsBlobStore.get(), rootId, [&result] (const BlockId& blockId) {
result.push_back(blockId);
});
cout << "done" << endl;
return result;
}
std::vector<BlockId> _getKnownBlockIds(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) {
auto knownBlobIds = _getKnownBlobIds(basedir, config, localStateDir);
auto blockStore = makeBlockStore(basedir, config, localStateDir);
auto nodeStore = make_unique_ref<DataNodeStore>(std::move(blockStore), config.configFile.config()->BlocksizeBytes());
std::vector<BlockId> result;
const uint32_t numNodes = nodeStore->numNodes();
result.reserve(numNodes);
uint32_t i = 0;
cout << "Listing all blocks used by these file system entities..." << endl;
for (const auto& blobId : knownBlobIds) {
_forEachBlockInBlob(nodeStore.get(), blobId, [&result, &i, numNodes] (const BlockId& blockId) {
cout << "\r" << (++i) << "/" << numNodes << flush;
result.push_back(blockId);
});
}
std::cout << "...done" << endl;
return result;
}
set<BlockId> _getAllBlockIds(const path& basedir, const CryConfigLoader::ConfigLoadResult& config, LocalStateDir& localStateDir) {
auto blockStore= makeBlockStore(basedir, config, localStateDir);
set<BlockId> result;
blockStore->forEachBlock([&result] (const BlockId& blockId) {
result.insert(blockId);
});
return result;
}
int main() {
int main(int argc, char* argv[]) {
if (argc != 2) {
std::cerr << "Usage: cryfs-stats [basedir]" << std::endl;
exit(1);
}
path basedir = argv[1];
std::cout << "Calculating stats for filesystem at " << basedir << std::endl;
auto console = std::make_shared<cpputils::IOStreamConsole>();
console->print("Loading config\n");
auto askPassword = [console] () {
return console->askPassword("Password: ");
};
auto keyProvider = make_unique_ref<CryPasswordBasedKeyProvider>(
unique_ref<CryKeyProvider> keyProvider = make_unique_ref<CryPasswordBasedKeyProvider>(
console,
askPassword,
askPassword,
make_unique_ref<SCrypt>(SCrypt::DefaultSettings)
);
auto config = CryConfigFile::load("/home/heinzi/basedir/cryfs.config", keyProvider.get());
set<BlockId> unaccountedBlocks = _getBlockstoreUnaccountedBlocks(*config->config());
//Remove all blocks that are referenced by a directory entry from unaccountedBlocks
set<BlockId> blocksReferencedByDirEntries = _getBlocksReferencedByDirEntries(*config->config());
for (const auto &blockId : blocksReferencedByDirEntries) {
unaccountedBlocks.erase(blockId);
auto config_path = basedir / "cryfs.config";
LocalStateDir localStateDir(cpputils::system::HomeDirectory::getXDGDataDir() / "cryfs");
CryConfigLoader config_loader(console, Random::OSRandom(), std::move(keyProvider), localStateDir, boost::none, boost::none, boost::none);
auto config = config_loader.load(config_path, false, true);
if (config == boost::none) {
std::cerr << "Error loading config file" << std::endl;
exit(1);
}
cout << "Listing all blocks..." << flush;
set<BlockId> unaccountedBlocks = _getAllBlockIds(basedir, *config, localStateDir);
cout << "done" << endl;
vector<BlockId> accountedBlocks = _getKnownBlockIds(basedir, *config, localStateDir);
for (const BlockId& blockId : accountedBlocks) {
auto num_erased = unaccountedBlocks.erase(blockId);
ASSERT(1 == num_erased, "Blob id referenced by directory entry but didn't found it on disk? This can't happen.");
}
console->print("Calculate statistics\n");
auto onDiskBlockStore = make_unique_ref<OnDiskBlockStore2>("/home/heinzi/basedir");
auto encryptedBlockStore = CryCiphers::find(config->config()->Cipher()).createEncryptedBlockstore(std::move(onDiskBlockStore), config->config()->EncryptionKey());
auto highLevelBlockStore = make_unique_ref<LowToHighLevelBlockStore>(std::move(encryptedBlockStore));
auto nodeStore = make_unique_ref<DataNodeStore>(std::move(highLevelBlockStore), config->config()->BlocksizeBytes());
auto blockStore = makeBlockStore(basedir, *config, localStateDir);
auto nodeStore = make_unique_ref<DataNodeStore>(std::move(blockStore), config->configFile.config()->BlocksizeBytes());
uint32_t numUnaccountedBlocks = unaccountedBlocks.size();
uint32_t numLeaves = 0;
uint32_t numInner = 0;
console->print("Unaccounted blocks: " + std::to_string(unaccountedBlocks.size()) + "\n");
for (const auto &blockId : unaccountedBlocks) {
console->print("\r" + std::to_string(numLeaves+numInner) + "/" + std::to_string(numUnaccountedBlocks));
console->print("\r" + std::to_string(numLeaves+numInner) + "/" + std::to_string(numUnaccountedBlocks) + ": ");
auto node = nodeStore->load(blockId);
auto innerNode = dynamic_pointer_move<DataInnerNode>(*node);
if (innerNode != none) {