From 4dbb380263544a8100cc327b760e8b17536495d1 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Tue, 8 Mar 2016 23:47:31 +0100 Subject: [PATCH] Decouple DirBlob from CryDevice --- .../implementations/onblocks/BlobStoreOnBlocks.cpp | 6 +++++- .../implementations/onblocks/BlobStoreOnBlocks.h | 5 +++-- .../onblocks/datanodestore/DataNodeStore.cpp | 6 +++++- .../onblocks/datanodestore/DataNodeStore.h | 5 +++-- .../onblocks/datanodestore/DataNodeView.h | 9 ++++----- .../onblocks/datatreestore/DataTree.cpp | 4 ++-- .../onblocks/datatreestore/DataTreeStore.h | 7 ++++++- .../ParallelAccessDataTreeStore.h | 3 ++- src/blobstore/interface/BlobStore.h | 2 ++ src/cryfs/filesystem/CryDevice.h | 4 ++-- src/cryfs/filesystem/fsblobstore/DirBlob.cpp | 10 +++++----- src/cryfs/filesystem/fsblobstore/DirBlob.h | 5 +++-- src/cryfs/filesystem/fsblobstore/FsBlobStore.cpp | 2 +- src/cryfs/filesystem/fsblobstore/FsBlobStore.h | 8 +++++++- 14 files changed, 50 insertions(+), 26 deletions(-) diff --git a/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp b/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp index e8dce0f9..ea054728 100644 --- a/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp +++ b/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp @@ -27,7 +27,7 @@ using datanodestore::DataNodeStore; using datatreestore::DataTreeStore; using parallelaccessdatatreestore::ParallelAccessDataTreeStore; -BlobStoreOnBlocks::BlobStoreOnBlocks(unique_ref blockStore, uint32_t blocksizeBytes) +BlobStoreOnBlocks::BlobStoreOnBlocks(unique_ref blockStore, uint64_t blocksizeBytes) : _dataTreeStore(make_unique_ref(make_unique_ref(make_unique_ref(make_unique_ref(std::move(blockStore)), blocksizeBytes)))) { } @@ -52,6 +52,10 @@ void BlobStoreOnBlocks::remove(unique_ref blob) { _dataTreeStore->remove((*_blob)->releaseTree()); } +uint64_t BlobStoreOnBlocks::blocksizeBytes() const { + return _dataTreeStore->blocksizeBytes(); +} + uint64_t BlobStoreOnBlocks::numBlocks() const { return _dataTreeStore->numNodes(); } diff --git a/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h b/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h index 27e0935b..b915c95b 100644 --- a/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h +++ b/src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h @@ -16,7 +16,7 @@ class ParallelAccessDataTreeStore; class BlobStoreOnBlocks final: public BlobStore { public: - BlobStoreOnBlocks(cpputils::unique_ref blockStore, uint32_t blocksizeBytes); + BlobStoreOnBlocks(cpputils::unique_ref blockStore, uint64_t blocksizeBytes); ~BlobStoreOnBlocks(); cpputils::unique_ref create() override; @@ -24,7 +24,8 @@ public: void remove(cpputils::unique_ref blob) override; - //TODO Test numBlocks/estimateSpaceForNumBlocksLeft + //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft + uint64_t blocksizeBytes() const override; uint64_t numBlocks() const override; uint64_t estimateSpaceForNumBlocksLeft() const override; diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp index 74103e5d..c61a8be1 100644 --- a/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp +++ b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp @@ -20,7 +20,7 @@ namespace blobstore { namespace onblocks { namespace datanodestore { -DataNodeStore::DataNodeStore(unique_ref blockstore, uint32_t blocksizeBytes) +DataNodeStore::DataNodeStore(unique_ref blockstore, uint64_t blocksizeBytes) : _blockstore(std::move(blockstore)), _layout(blocksizeBytes) { } @@ -96,6 +96,10 @@ uint64_t DataNodeStore::estimateSpaceForNumNodesLeft() const { return _blockstore->estimateNumFreeBytes() / _layout.blocksizeBytes(); } +uint64_t DataNodeStore::blocksizeBytes() const { + return _layout.blocksizeBytes(); +} + void DataNodeStore::removeSubtree(unique_ref node) { //TODO Make this faster by not loading the leaves but just deleting them. Can be recognized, because of the depth of their parents. DataInnerNode *inner = dynamic_cast(node.get()); diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h index 88860ba7..4cbe2339 100644 --- a/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h +++ b/src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h @@ -21,7 +21,7 @@ class DataInnerNode; class DataNodeStore final { public: - DataNodeStore(cpputils::unique_ref blockstore, uint32_t blocksizeBytes); + DataNodeStore(cpputils::unique_ref blockstore, uint64_t blocksizeBytes); ~DataNodeStore(); static constexpr uint8_t MAX_DEPTH = 10; @@ -41,7 +41,8 @@ public: void removeSubtree(cpputils::unique_ref node); - //TODO Test numBlocks/estimateSpaceForNumBlocksLeft + //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft + uint64_t blocksizeBytes() const; uint64_t numNodes() const; uint64_t estimateSpaceForNumNodesLeft() const; //TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree() diff --git a/src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h b/src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h index 834061b3..ab69bbfc 100644 --- a/src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h +++ b/src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h @@ -19,7 +19,7 @@ namespace datanodestore { //TODO Move DataNodeLayout into own file class DataNodeLayout final { public: - constexpr DataNodeLayout(uint32_t blocksizeBytes) + constexpr DataNodeLayout(uint64_t blocksizeBytes) :_blocksizeBytes( (HEADERSIZE_BYTES + 2*sizeof(DataInnerNode_ChildEntry) <= blocksizeBytes) ? blocksizeBytes @@ -37,22 +37,21 @@ public: //Size of a block (header + data region) - constexpr uint32_t blocksizeBytes() const { + constexpr uint64_t blocksizeBytes() const { return _blocksizeBytes; } //Number of bytes in the data region of a node - constexpr uint32_t datasizeBytes() const { + constexpr uint64_t datasizeBytes() const { return _blocksizeBytes - HEADERSIZE_BYTES; } //Maximum number of children an inner node can store - constexpr uint32_t maxChildrenPerInnerNode() const { + constexpr uint64_t maxChildrenPerInnerNode() const { return datasizeBytes() / sizeof(DataInnerNode_ChildEntry); } //Maximum number of bytes a leaf can store - //We are returning uint64_t here, because calculations involving maxBytesPerLeaf most probably should use 64bit integers to support blobs >4GB. constexpr uint64_t maxBytesPerLeaf() const { return datasizeBytes(); } diff --git a/src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp b/src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp index d2d87049..682ff062 100644 --- a/src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp +++ b/src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp @@ -166,7 +166,7 @@ void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, functionlayout().maxChildrenPerInnerNode(), endIndex); + uint8_t neededTreeDepth = utils::ceilLog(_nodeStore->layout().maxChildrenPerInnerNode(), (uint64_t)endIndex); uint32_t numLeaves = this->_numLeaves(*_rootNode); // TODO Querying the size causes a tree traversal down to the leaves. Possible without querying the size? if (_rootNode->depth() < neededTreeDepth) { //TODO Test cases that actually increase it here by 0 level / 1 level / more than 1 level @@ -250,7 +250,7 @@ unique_ref DataTree::addChildTo(DataInnerNode *node) { } uint32_t DataTree::leavesPerFullChild(const DataInnerNode &root) const { - return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), (uint32_t)root.depth()-1); + return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), (uint64_t)root.depth()-1); } uint64_t DataTree::numStoredBytes() const { diff --git a/src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h b/src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h index ebc5e06d..4e253728 100644 --- a/src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h +++ b/src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h @@ -25,7 +25,8 @@ public: void remove(cpputils::unique_ref tree); - //TODO Test numBlocks/estimateSpaceForNumBlocksLeft + //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft + uint64_t blocksizeBytes() const; uint64_t numNodes() const; uint64_t estimateSpaceForNumNodesLeft() const; @@ -43,6 +44,10 @@ inline uint64_t DataTreeStore::estimateSpaceForNumNodesLeft() const { return _nodeStore->estimateSpaceForNumNodesLeft(); } +inline uint64_t DataTreeStore::blocksizeBytes() const { + return _nodeStore->blocksizeBytes(); +} + } } } diff --git a/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h index 2f65fb67..1ff4c069 100644 --- a/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h +++ b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h @@ -26,7 +26,8 @@ public: void remove(cpputils::unique_ref tree); - //TODO Test numBlocks/estimateSpaceForNumBlocksLeft + //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft + uint64_t blocksizeBytes() const; uint64_t numNodes() const; uint64_t estimateSpaceForNumNodesLeft() const; diff --git a/src/blobstore/interface/BlobStore.h b/src/blobstore/interface/BlobStore.h index 5cfe1a29..bf7c0653 100644 --- a/src/blobstore/interface/BlobStore.h +++ b/src/blobstore/interface/BlobStore.h @@ -11,6 +11,7 @@ namespace blobstore { +//TODO Remove this interface. We'll only use BlobStoreOnBlocks and never a different one. Rename BlobStoreOnBlocks to simply BlobStore. class BlobStore { public: virtual ~BlobStore() {} @@ -21,6 +22,7 @@ public: virtual uint64_t numBlocks() const = 0; virtual uint64_t estimateSpaceForNumBlocksLeft() const = 0; + virtual uint64_t blocksizeBytes() const = 0; }; } diff --git a/src/cryfs/filesystem/CryDevice.h b/src/cryfs/filesystem/CryDevice.h index 72123e8f..c448ef21 100644 --- a/src/cryfs/filesystem/CryDevice.h +++ b/src/cryfs/filesystem/CryDevice.h @@ -17,8 +17,6 @@ namespace cryfs { class CryDevice final: public fspp::Device { public: - static constexpr uint32_t BLOCKSIZE_BYTES = 32 * 1024; - CryDevice(CryConfigFile config, cpputils::unique_ref blockStore); void statfs(const boost::filesystem::path &path, struct ::statvfs *fsstat) override; @@ -39,6 +37,8 @@ public: private: + static constexpr uint32_t BLOCKSIZE_BYTES = 32 * 1024; + cpputils::unique_ref _fsBlobStore; blockstore::Key _rootKey; diff --git a/src/cryfs/filesystem/fsblobstore/DirBlob.cpp b/src/cryfs/filesystem/fsblobstore/DirBlob.cpp index 25d0c160..23101a2a 100644 --- a/src/cryfs/filesystem/fsblobstore/DirBlob.cpp +++ b/src/cryfs/filesystem/fsblobstore/DirBlob.cpp @@ -27,8 +27,8 @@ namespace fsblobstore { constexpr off_t DirBlob::DIR_LSTAT_SIZE; -DirBlob::DirBlob(unique_ref blob, std::function getLstatSize) : - FsBlob(std::move(blob)), _getLstatSize(getLstatSize), _entries(), _mutex(), _changed(false) { +DirBlob::DirBlob(FsBlobStore *fsBlobStore, unique_ref blob, std::function getLstatSize) : + FsBlob(std::move(blob)), _fsBlobStore(fsBlobStore), _getLstatSize(getLstatSize), _entries(), _mutex(), _changed(false) { ASSERT(baseBlob().blobType() == FsBlobView::BlobType::DIR, "Loaded blob is not a directory"); _readEntriesFromBlob(); } @@ -44,9 +44,9 @@ void DirBlob::flush() { baseBlob().flush(); } -unique_ref DirBlob::InitializeEmptyDir(unique_ref blob, std::function getLstatSize) { +unique_ref DirBlob::InitializeEmptyDir(FsBlobStore *fsBlobStore, unique_ref blob, std::function getLstatSize) { InitializeBlob(blob.get(), FsBlobView::BlobType::DIR); - return make_unique_ref(std::move(blob), getLstatSize); + return make_unique_ref(fsBlobStore, std::move(blob), getLstatSize); } void DirBlob::_writeEntriesToBlob() { @@ -136,7 +136,7 @@ void DirBlob::statChild(const Key &key, struct ::stat *result) const { result->st_size = _getLstatSize(key); //TODO Move ceilDivision to general utils which can be used by cryfs as well result->st_blocks = blobstore::onblocks::utils::ceilDivision(result->st_size, (off_t)512); - result->st_blksize = CryDevice::BLOCKSIZE_BYTES; //TODO FsBlobStore::BLOCKSIZE_BYTES would be cleaner + result->st_blksize = _fsBlobStore->blocksizeBytes(); } void DirBlob::chmodChild(const Key &key, mode_t mode) { diff --git a/src/cryfs/filesystem/fsblobstore/DirBlob.h b/src/cryfs/filesystem/fsblobstore/DirBlob.h index 6a32331f..92684a89 100644 --- a/src/cryfs/filesystem/fsblobstore/DirBlob.h +++ b/src/cryfs/filesystem/fsblobstore/DirBlob.h @@ -17,10 +17,10 @@ namespace cryfs { public: constexpr static off_t DIR_LSTAT_SIZE = 4096; - static cpputils::unique_ref InitializeEmptyDir(cpputils::unique_ref blob, + static cpputils::unique_ref InitializeEmptyDir(FsBlobStore *fsBlobStore, cpputils::unique_ref blob, std::function getLstatSize); - DirBlob(cpputils::unique_ref blob, std::function getLstatSize); + DirBlob(FsBlobStore *fsBlobStore, cpputils::unique_ref blob, std::function getLstatSize); ~DirBlob(); @@ -68,6 +68,7 @@ namespace cryfs { cpputils::unique_ref releaseBaseBlob() override; + FsBlobStore *_fsBlobStore; std::function _getLstatSize; DirEntryList _entries; mutable std::mutex _mutex; diff --git a/src/cryfs/filesystem/fsblobstore/FsBlobStore.cpp b/src/cryfs/filesystem/fsblobstore/FsBlobStore.cpp index 288f50ad..7200b0fc 100644 --- a/src/cryfs/filesystem/fsblobstore/FsBlobStore.cpp +++ b/src/cryfs/filesystem/fsblobstore/FsBlobStore.cpp @@ -23,7 +23,7 @@ boost::optional> FsBlobStore::load(const blockstore::Key &key if (blobType == FsBlobView::BlobType::FILE) { return unique_ref(make_unique_ref(std::move(*blob))); } else if (blobType == FsBlobView::BlobType::DIR) { - return unique_ref(make_unique_ref(std::move(*blob), _getLstatSize())); + return unique_ref(make_unique_ref(this, std::move(*blob), _getLstatSize())); } else if (blobType == FsBlobView::BlobType::SYMLINK) { return unique_ref(make_unique_ref(std::move(*blob))); } else { diff --git a/src/cryfs/filesystem/fsblobstore/FsBlobStore.h b/src/cryfs/filesystem/fsblobstore/FsBlobStore.h index 9dd5d589..583f56b0 100644 --- a/src/cryfs/filesystem/fsblobstore/FsBlobStore.h +++ b/src/cryfs/filesystem/fsblobstore/FsBlobStore.h @@ -25,6 +25,8 @@ namespace cryfs { uint64_t numBlocks() const; uint64_t estimateSpaceForNumBlocksLeft() const; + uint64_t blocksizeBytes() const; + private: std::function _getLstatSize(); @@ -45,7 +47,7 @@ namespace cryfs { inline cpputils::unique_ref FsBlobStore::createDirBlob() { auto blob = _baseBlobStore->create(); - return DirBlob::InitializeEmptyDir(std::move(blob), _getLstatSize()); + return DirBlob::InitializeEmptyDir(this, std::move(blob), _getLstatSize()); } inline cpputils::unique_ref FsBlobStore::createSymlinkBlob(const boost::filesystem::path &target) { @@ -72,6 +74,10 @@ namespace cryfs { return (*blob)->lstat_size(); }; } + + inline uint64_t FsBlobStore::blocksizeBytes() const { + return _baseBlobStore->blocksizeBytes(); + } } }