hardcoresushi/libcryfs
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Decouple DirBlob from CryDevice

Browse Source
This commit is contained in:
Sebastian Messmer 2016-03-08 23:47:31 +01:00
parent 526b749d1d
commit 4dbb380263
14 changed files with 50 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp
View File

@ -27,7 +27,7 @@ using datanodestore::DataNodeStore;
using datatreestore::DataTreeStore; using datatreestore::DataTreeStore;
using parallelaccessdatatreestore::ParallelAccessDataTreeStore; using parallelaccessdatatreestore::ParallelAccessDataTreeStore;
BlobStoreOnBlocks::BlobStoreOnBlocks(unique_ref<BlockStore> blockStore, uint32_t blocksizeBytes) BlobStoreOnBlocks::BlobStoreOnBlocks(unique_ref<BlockStore> blockStore, uint64_t blocksizeBytes)
: _dataTreeStore(make_unique_ref<ParallelAccessDataTreeStore>(make_unique_ref<DataTreeStore>(make_unique_ref<DataNodeStore>(make_unique_ref<ParallelAccessBlockStore>(std::move(blockStore)), blocksizeBytes)))) { : _dataTreeStore(make_unique_ref<ParallelAccessDataTreeStore>(make_unique_ref<DataTreeStore>(make_unique_ref<DataNodeStore>(make_unique_ref<ParallelAccessBlockStore>(std::move(blockStore)), blocksizeBytes)))) {
} }
@ -52,6 +52,10 @@ void BlobStoreOnBlocks::remove(unique_ref<Blob> blob) {
_dataTreeStore->remove((*_blob)->releaseTree()); _dataTreeStore->remove((*_blob)->releaseTree());
} }
uint64_t BlobStoreOnBlocks::blocksizeBytes() const {
return _dataTreeStore->blocksizeBytes();
}
uint64_t BlobStoreOnBlocks::numBlocks() const { uint64_t BlobStoreOnBlocks::numBlocks() const {
return _dataTreeStore->numNodes(); return _dataTreeStore->numNodes();
} }

5
src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h
View File

@ -16,7 +16,7 @@ class ParallelAccessDataTreeStore;
class BlobStoreOnBlocks final: public BlobStore { class BlobStoreOnBlocks final: public BlobStore {
public: public:
BlobStoreOnBlocks(cpputils::unique_ref<blockstore::BlockStore> blockStore, uint32_t blocksizeBytes); BlobStoreOnBlocks(cpputils::unique_ref<blockstore::BlockStore> blockStore, uint64_t blocksizeBytes);
~BlobStoreOnBlocks(); ~BlobStoreOnBlocks();
cpputils::unique_ref<Blob> create() override; cpputils::unique_ref<Blob> create() override;
@ -24,7 +24,8 @@ public:
void remove(cpputils::unique_ref<Blob> blob) override; void remove(cpputils::unique_ref<Blob> blob) override;
//TODO Test numBlocks/estimateSpaceForNumBlocksLeft //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
uint64_t blocksizeBytes() const override;
uint64_t numBlocks() const override; uint64_t numBlocks() const override;
uint64_t estimateSpaceForNumBlocksLeft() const override; uint64_t estimateSpaceForNumBlocksLeft() const override;

6
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp
View File

@ -20,7 +20,7 @@ namespace blobstore {
namespace onblocks { namespace onblocks {
namespace datanodestore { namespace datanodestore {
DataNodeStore::DataNodeStore(unique_ref<BlockStore> blockstore, uint32_t blocksizeBytes) DataNodeStore::DataNodeStore(unique_ref<BlockStore> blockstore, uint64_t blocksizeBytes)
: _blockstore(std::move(blockstore)), _layout(blocksizeBytes) { : _blockstore(std::move(blockstore)), _layout(blocksizeBytes) {
} }
@ -96,6 +96,10 @@ uint64_t DataNodeStore::estimateSpaceForNumNodesLeft() const {
return _blockstore->estimateNumFreeBytes() / _layout.blocksizeBytes(); return _blockstore->estimateNumFreeBytes() / _layout.blocksizeBytes();
} }
uint64_t DataNodeStore::blocksizeBytes() const {
return _layout.blocksizeBytes();
}
void DataNodeStore::removeSubtree(unique_ref<DataNode> node) { void DataNodeStore::removeSubtree(unique_ref<DataNode> node) {
//TODO Make this faster by not loading the leaves but just deleting them. Can be recognized, because of the depth of their parents. //TODO Make this faster by not loading the leaves but just deleting them. Can be recognized, because of the depth of their parents.
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(node.get()); DataInnerNode *inner = dynamic_cast<DataInnerNode*>(node.get());

5
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h
View File

@ -21,7 +21,7 @@ class DataInnerNode;
class DataNodeStore final { class DataNodeStore final {
public: public:
DataNodeStore(cpputils::unique_ref<blockstore::BlockStore> blockstore, uint32_t blocksizeBytes); DataNodeStore(cpputils::unique_ref<blockstore::BlockStore> blockstore, uint64_t blocksizeBytes);
~DataNodeStore(); ~DataNodeStore();
static constexpr uint8_t MAX_DEPTH = 10; static constexpr uint8_t MAX_DEPTH = 10;
@ -41,7 +41,8 @@ public:
void removeSubtree(cpputils::unique_ref<DataNode> node); void removeSubtree(cpputils::unique_ref<DataNode> node);
//TODO Test numBlocks/estimateSpaceForNumBlocksLeft //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
uint64_t blocksizeBytes() const;
uint64_t numNodes() const; uint64_t numNodes() const;
uint64_t estimateSpaceForNumNodesLeft() const; uint64_t estimateSpaceForNumNodesLeft() const;
//TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree() //TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree()

9
src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h
View File

@ -19,7 +19,7 @@ namespace datanodestore {
//TODO Move DataNodeLayout into own file //TODO Move DataNodeLayout into own file
class DataNodeLayout final { class DataNodeLayout final {
public: public:
constexpr DataNodeLayout(uint32_t blocksizeBytes) constexpr DataNodeLayout(uint64_t blocksizeBytes)
:_blocksizeBytes( :_blocksizeBytes(
(HEADERSIZE_BYTES + 2*sizeof(DataInnerNode_ChildEntry) <= blocksizeBytes) (HEADERSIZE_BYTES + 2*sizeof(DataInnerNode_ChildEntry) <= blocksizeBytes)
? blocksizeBytes ? blocksizeBytes
@ -37,22 +37,21 @@ public:
//Size of a block (header + data region) //Size of a block (header + data region)
constexpr uint32_t blocksizeBytes() const { constexpr uint64_t blocksizeBytes() const {
return _blocksizeBytes; return _blocksizeBytes;
} }
//Number of bytes in the data region of a node //Number of bytes in the data region of a node
constexpr uint32_t datasizeBytes() const { constexpr uint64_t datasizeBytes() const {
return _blocksizeBytes - HEADERSIZE_BYTES; return _blocksizeBytes - HEADERSIZE_BYTES;
} }
//Maximum number of children an inner node can store //Maximum number of children an inner node can store
constexpr uint32_t maxChildrenPerInnerNode() const { constexpr uint64_t maxChildrenPerInnerNode() const {
return datasizeBytes() / sizeof(DataInnerNode_ChildEntry); return datasizeBytes() / sizeof(DataInnerNode_ChildEntry);
} }
//Maximum number of bytes a leaf can store //Maximum number of bytes a leaf can store
//We are returning uint64_t here, because calculations involving maxBytesPerLeaf most probably should use 64bit integers to support blobs >4GB.
constexpr uint64_t maxBytesPerLeaf() const { constexpr uint64_t maxBytesPerLeaf() const {
return datasizeBytes(); return datasizeBytes();
} }

4
src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp
View File

@ -166,7 +166,7 @@ void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<v
return; return;
} }
uint8_t neededTreeDepth = utils::ceilLog(_nodeStore->layout().maxChildrenPerInnerNode(), endIndex); uint8_t neededTreeDepth = utils::ceilLog(_nodeStore->layout().maxChildrenPerInnerNode(), (uint64_t)endIndex);
uint32_t numLeaves = this->_numLeaves(*_rootNode); // TODO Querying the size causes a tree traversal down to the leaves. Possible without querying the size? uint32_t numLeaves = this->_numLeaves(*_rootNode); // TODO Querying the size causes a tree traversal down to the leaves. Possible without querying the size?
if (_rootNode->depth() < neededTreeDepth) { if (_rootNode->depth() < neededTreeDepth) {
//TODO Test cases that actually increase it here by 0 level / 1 level / more than 1 level //TODO Test cases that actually increase it here by 0 level / 1 level / more than 1 level
@ -250,7 +250,7 @@ unique_ref<DataNode> DataTree::addChildTo(DataInnerNode *node) {
} }
uint32_t DataTree::leavesPerFullChild(const DataInnerNode &root) const { uint32_t DataTree::leavesPerFullChild(const DataInnerNode &root) const {
return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), (uint32_t)root.depth()-1); return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), (uint64_t)root.depth()-1);
} }
uint64_t DataTree::numStoredBytes() const { uint64_t DataTree::numStoredBytes() const {

7
src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h
View File

@ -25,7 +25,8 @@ public:
void remove(cpputils::unique_ref<DataTree> tree); void remove(cpputils::unique_ref<DataTree> tree);
//TODO Test numBlocks/estimateSpaceForNumBlocksLeft //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
uint64_t blocksizeBytes() const;
uint64_t numNodes() const; uint64_t numNodes() const;
uint64_t estimateSpaceForNumNodesLeft() const; uint64_t estimateSpaceForNumNodesLeft() const;
@ -43,6 +44,10 @@ inline uint64_t DataTreeStore::estimateSpaceForNumNodesLeft() const {
return _nodeStore->estimateSpaceForNumNodesLeft(); return _nodeStore->estimateSpaceForNumNodesLeft();
} }
inline uint64_t DataTreeStore::blocksizeBytes() const {
return _nodeStore->blocksizeBytes();
}
} }
} }
} }

3
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h
View File

@ -26,7 +26,8 @@ public:
void remove(cpputils::unique_ref<DataTreeRef> tree); void remove(cpputils::unique_ref<DataTreeRef> tree);
//TODO Test numBlocks/estimateSpaceForNumBlocksLeft //TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
uint64_t blocksizeBytes() const;
uint64_t numNodes() const; uint64_t numNodes() const;
uint64_t estimateSpaceForNumNodesLeft() const; uint64_t estimateSpaceForNumNodesLeft() const;

2
src/blobstore/interface/BlobStore.h
View File

@ -11,6 +11,7 @@
namespace blobstore { namespace blobstore {
//TODO Remove this interface. We'll only use BlobStoreOnBlocks and never a different one. Rename BlobStoreOnBlocks to simply BlobStore.
class BlobStore { class BlobStore {
public: public:
virtual ~BlobStore() {} virtual ~BlobStore() {}
@ -21,6 +22,7 @@ public:
virtual uint64_t numBlocks() const = 0; virtual uint64_t numBlocks() const = 0;
virtual uint64_t estimateSpaceForNumBlocksLeft() const = 0; virtual uint64_t estimateSpaceForNumBlocksLeft() const = 0;
virtual uint64_t blocksizeBytes() const = 0;
}; };
} }

4
src/cryfs/filesystem/CryDevice.h
View File

@ -17,8 +17,6 @@ namespace cryfs {
class CryDevice final: public fspp::Device { class CryDevice final: public fspp::Device {
public: public:
static constexpr uint32_t BLOCKSIZE_BYTES = 32 * 1024;
CryDevice(CryConfigFile config, cpputils::unique_ref<blockstore::BlockStore> blockStore); CryDevice(CryConfigFile config, cpputils::unique_ref<blockstore::BlockStore> blockStore);
void statfs(const boost::filesystem::path &path, struct ::statvfs *fsstat) override; void statfs(const boost::filesystem::path &path, struct ::statvfs *fsstat) override;
@ -39,6 +37,8 @@ public:
private: private:
static constexpr uint32_t BLOCKSIZE_BYTES = 32 * 1024;
cpputils::unique_ref<parallelaccessfsblobstore::ParallelAccessFsBlobStore> _fsBlobStore; cpputils::unique_ref<parallelaccessfsblobstore::ParallelAccessFsBlobStore> _fsBlobStore;
blockstore::Key _rootKey; blockstore::Key _rootKey;

10
src/cryfs/filesystem/fsblobstore/DirBlob.cpp
View File

@ -27,8 +27,8 @@ namespace fsblobstore {
constexpr off_t DirBlob::DIR_LSTAT_SIZE; constexpr off_t DirBlob::DIR_LSTAT_SIZE;
DirBlob::DirBlob(unique_ref<Blob> blob, std::function<off_t (const blockstore::Key&)> getLstatSize) : DirBlob::DirBlob(FsBlobStore *fsBlobStore, unique_ref<Blob> blob, std::function<off_t (const blockstore::Key&)> getLstatSize) :
FsBlob(std::move(blob)), _getLstatSize(getLstatSize), _entries(), _mutex(), _changed(false) { FsBlob(std::move(blob)), _fsBlobStore(fsBlobStore), _getLstatSize(getLstatSize), _entries(), _mutex(), _changed(false) {
ASSERT(baseBlob().blobType() == FsBlobView::BlobType::DIR, "Loaded blob is not a directory"); ASSERT(baseBlob().blobType() == FsBlobView::BlobType::DIR, "Loaded blob is not a directory");
_readEntriesFromBlob(); _readEntriesFromBlob();
} }
@ -44,9 +44,9 @@ void DirBlob::flush() {
baseBlob().flush(); baseBlob().flush();
} }
unique_ref<DirBlob> DirBlob::InitializeEmptyDir(unique_ref<Blob> blob, std::function<off_t(const blockstore::Key&)> getLstatSize) { unique_ref<DirBlob> DirBlob::InitializeEmptyDir(FsBlobStore *fsBlobStore, unique_ref<Blob> blob, std::function<off_t(const blockstore::Key&)> getLstatSize) {
InitializeBlob(blob.get(), FsBlobView::BlobType::DIR); InitializeBlob(blob.get(), FsBlobView::BlobType::DIR);
return make_unique_ref<DirBlob>(std::move(blob), getLstatSize); return make_unique_ref<DirBlob>(fsBlobStore, std::move(blob), getLstatSize);
} }
void DirBlob::_writeEntriesToBlob() { void DirBlob::_writeEntriesToBlob() {
@ -136,7 +136,7 @@ void DirBlob::statChild(const Key &key, struct ::stat *result) const {
result->st_size = _getLstatSize(key); result->st_size = _getLstatSize(key);
//TODO Move ceilDivision to general utils which can be used by cryfs as well //TODO Move ceilDivision to general utils which can be used by cryfs as well
result->st_blocks = blobstore::onblocks::utils::ceilDivision(result->st_size, (off_t)512); result->st_blocks = blobstore::onblocks::utils::ceilDivision(result->st_size, (off_t)512);
result->st_blksize = CryDevice::BLOCKSIZE_BYTES; //TODO FsBlobStore::BLOCKSIZE_BYTES would be cleaner result->st_blksize = _fsBlobStore->blocksizeBytes();
} }
void DirBlob::chmodChild(const Key &key, mode_t mode) { void DirBlob::chmodChild(const Key &key, mode_t mode) {

5
src/cryfs/filesystem/fsblobstore/DirBlob.h
View File

@ -17,10 +17,10 @@ namespace cryfs {
public: public:
constexpr static off_t DIR_LSTAT_SIZE = 4096; constexpr static off_t DIR_LSTAT_SIZE = 4096;
static cpputils::unique_ref<DirBlob> InitializeEmptyDir(cpputils::unique_ref<blobstore::Blob> blob, static cpputils::unique_ref<DirBlob> InitializeEmptyDir(FsBlobStore *fsBlobStore, cpputils::unique_ref<blobstore::Blob> blob,
std::function<off_t (const blockstore::Key&)> getLstatSize); std::function<off_t (const blockstore::Key&)> getLstatSize);
DirBlob(cpputils::unique_ref<blobstore::Blob> blob, std::function<off_t (const blockstore::Key&)> getLstatSize); DirBlob(FsBlobStore *fsBlobStore, cpputils::unique_ref<blobstore::Blob> blob, std::function<off_t (const blockstore::Key&)> getLstatSize);
~DirBlob(); ~DirBlob();
@ -68,6 +68,7 @@ namespace cryfs {
cpputils::unique_ref<blobstore::Blob> releaseBaseBlob() override; cpputils::unique_ref<blobstore::Blob> releaseBaseBlob() override;
FsBlobStore *_fsBlobStore;
std::function<off_t (const blockstore::Key&)> _getLstatSize; std::function<off_t (const blockstore::Key&)> _getLstatSize;
DirEntryList _entries; DirEntryList _entries;
mutable std::mutex _mutex; mutable std::mutex _mutex;

2
src/cryfs/filesystem/fsblobstore/FsBlobStore.cpp
View File

@ -23,7 +23,7 @@ boost::optional<unique_ref<FsBlob>> FsBlobStore::load(const blockstore::Key &key
if (blobType == FsBlobView::BlobType::FILE) { if (blobType == FsBlobView::BlobType::FILE) {
return unique_ref<FsBlob>(make_unique_ref<FileBlob>(std::move(*blob))); return unique_ref<FsBlob>(make_unique_ref<FileBlob>(std::move(*blob)));
} else if (blobType == FsBlobView::BlobType::DIR) { } else if (blobType == FsBlobView::BlobType::DIR) {
return unique_ref<FsBlob>(make_unique_ref<DirBlob>(std::move(*blob), _getLstatSize())); return unique_ref<FsBlob>(make_unique_ref<DirBlob>(this, std::move(*blob), _getLstatSize()));
} else if (blobType == FsBlobView::BlobType::SYMLINK) { } else if (blobType == FsBlobView::BlobType::SYMLINK) {
return unique_ref<FsBlob>(make_unique_ref<SymlinkBlob>(std::move(*blob))); return unique_ref<FsBlob>(make_unique_ref<SymlinkBlob>(std::move(*blob)));
} else { } else {

8
src/cryfs/filesystem/fsblobstore/FsBlobStore.h
View File

@ -25,6 +25,8 @@ namespace cryfs {
uint64_t numBlocks() const; uint64_t numBlocks() const;
uint64_t estimateSpaceForNumBlocksLeft() const; uint64_t estimateSpaceForNumBlocksLeft() const;
uint64_t blocksizeBytes() const;
private: private:
std::function<off_t(const blockstore::Key &)> _getLstatSize(); std::function<off_t(const blockstore::Key &)> _getLstatSize();
@ -45,7 +47,7 @@ namespace cryfs {
inline cpputils::unique_ref<DirBlob> FsBlobStore::createDirBlob() { inline cpputils::unique_ref<DirBlob> FsBlobStore::createDirBlob() {
auto blob = _baseBlobStore->create(); auto blob = _baseBlobStore->create();
return DirBlob::InitializeEmptyDir(std::move(blob), _getLstatSize()); return DirBlob::InitializeEmptyDir(this, std::move(blob), _getLstatSize());
} }
inline cpputils::unique_ref<SymlinkBlob> FsBlobStore::createSymlinkBlob(const boost::filesystem::path &target) { inline cpputils::unique_ref<SymlinkBlob> FsBlobStore::createSymlinkBlob(const boost::filesystem::path &target) {
@ -72,6 +74,10 @@ namespace cryfs {
return (*blob)->lstat_size(); return (*blob)->lstat_size();
}; };
} }
inline uint64_t FsBlobStore::blocksizeBytes() const {
return _baseBlobStore->blocksizeBytes();
}
} }
} }