hardcoresushi/libcryfs
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Further reduce number of nodes loaded when deleting a tree and write test cases for it

Browse Source
This commit is contained in:
Sebastian Messmer 2016-07-15 15:42:34 +02:00
parent 6fc62a58fa
commit b1b90b8c3d
6 changed files with 249 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp
View File

@ -30,7 +30,6 @@ DataNodeStore::~DataNodeStore() {
}
unique_ref<DataNode> DataNodeStore::load(unique_ref<Block> block) {
ASSERT(block->size() == _layout.blocksizeBytes(), "Loading block of wrong size");
DataNodeView node(std::move(block));
if (node.Depth() == 0) {
@ -56,6 +55,7 @@ optional<unique_ref<DataNode>> DataNodeStore::load(const Key &key) {
if (block == none) {
return none;
} else {
ASSERT((*block)->size() == _layout.blocksizeBytes(), "Loading block of wrong size");
return load(std::move(*block));
}
}
@ -70,14 +70,10 @@ unique_ref<DataNode> DataNodeStore::overwriteNodeWith(unique_ref<DataNode> targe
ASSERT(target->node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Target node has wrong layout. Is it from the same DataNodeStore?");
ASSERT(source.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
Key key = target->key();
{
auto targetBlock = target->node().releaseBlock();
cpputils::destruct(std::move(target)); // Call destructor
blockstore::utils::copyTo(targetBlock.get(), source.node().block());
}
auto loaded = load(key);
ASSERT(loaded != none, "Couldn't load the target node after overwriting it");
return std::move(*loaded);
auto targetBlock = target->node().releaseBlock();
cpputils::destruct(std::move(target)); // Call destructor
blockstore::utils::copyTo(targetBlock.get(), source.node().block());
return DataNodeStore::load(std::move(targetBlock));
}
void DataNodeStore::remove(unique_ref<DataNode> node) {
@ -90,36 +86,36 @@ void DataNodeStore::remove(const Key &key) {
_blockstore->remove(key);
}
void DataNodeStore::removeSubtree(const Key &key) {
auto node = load(key);
ASSERT(node != none, "Node for removeSubtree not found");
auto inner = dynamic_pointer_move<DataInnerNode>(*node);
if (inner == none) {
ASSERT((*node)->depth() == 0, "If it's not an inner node, it has to be a leaf.");
remove(std::move(*node));
} else {
_removeSubtree(std::move(*inner));
void DataNodeStore::removeSubtree(unique_ref<DataNode> node) {
auto leaf = dynamic_pointer_move<DataLeafNode>(node);
if (leaf != none) {
remove(std::move(*leaf));
return;
}
auto inner = dynamic_pointer_move<DataInnerNode>(node);
ASSERT(inner != none, "Is neither a leaf nor an inner node");
for (uint32_t i = 0; i < (*inner)->numChildren(); ++i) {
removeSubtree((*inner)->depth()-1, (*inner)->getChild(i)->key());
}
remove(std::move(*inner));
}
void DataNodeStore::_removeSubtree(cpputils::unique_ref<DataInnerNode> node) {
if (node->depth() == 1) {
for (uint32_t i = 0; i < node->numChildren(); ++i) {
remove(node->getChild(i)->key());
}
void DataNodeStore::removeSubtree(uint8_t depth, const Key &key) {
if (depth == 0) {
remove(key);
} else {
ASSERT(node->depth() > 1, "This if branch is only called when our children are inner nodes.");
for (uint32_t i = 0; i < node->numChildren(); ++i) {
auto child = load(node->getChild(i)->key());
ASSERT(child != none, "Child not found");
auto inner = dynamic_pointer_move<DataInnerNode>(*child);
ASSERT(inner != none, "Expected inner node as child");
_removeSubtree(std::move(*inner));
auto node = load(key);
ASSERT(node != none, "Node for removeSubtree not found");
auto inner = dynamic_pointer_move<DataInnerNode>(*node);
ASSERT(inner != none, "Is not an inner node, but depth was not zero");
ASSERT((*inner)->depth() == depth, "Wrong depth given");
for (uint32_t i = 0; i < (*inner)->numChildren(); ++i) {
removeSubtree(depth-1, (*inner)->getChild(i)->key());
}
remove(std::move(*inner));
}
remove(std::move(node));
}
uint64_t DataNodeStore::numNodes() const {

6
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h
View File

@ -29,6 +29,7 @@ public:
DataNodeLayout layout() const;
boost::optional<cpputils::unique_ref<DataNode>> load(const blockstore::Key &key);
static cpputils::unique_ref<DataNode> load(cpputils::unique_ref<blockstore::Block> block);
cpputils::unique_ref<DataLeafNode> createNewLeafNode(cpputils::Data data);
cpputils::unique_ref<DataInnerNode> createNewInnerNode(uint8_t depth, const std::vector<blockstore::Key> &children);
@ -39,7 +40,8 @@ public:
void remove(cpputils::unique_ref<DataNode> node);
void remove(const blockstore::Key &key);
void removeSubtree(const blockstore::Key &key);
void removeSubtree(uint8_t depth, const blockstore::Key &key);
void removeSubtree(cpputils::unique_ref<DataNode> node);
//TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
uint64_t virtualBlocksizeBytes() const;
@ -48,8 +50,6 @@ public:
//TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree()
private:
cpputils::unique_ref<DataNode> load(cpputils::unique_ref<blockstore::Block> block);
void _removeSubtree(cpputils::unique_ref<DataInnerNode> node);
cpputils::unique_ref<blockstore::BlockStore> _blockstore;
const DataNodeLayout _layout;

11
src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp
View File

@ -153,7 +153,9 @@ void DataTree::resizeNumBytes(uint64_t newNumBytes) {
uint32_t maxChildrenPerInnerNode = _nodeStore->layout().maxChildrenPerInnerNode();
auto onExistingLeaf = [newLastLeafSize] (uint32_t /*index*/, datanodestore::DataLeafNode* leaf) {
// This is only called, if the new last leaf was already existing
leaf->resize(newLastLeafSize);
if (leaf->numBytes() != newLastLeafSize) {
leaf->resize(newLastLeafSize);
}
};
auto onCreateLeaf = [newLastLeafSize] (uint32_t /*index*/) -> Data {
// This is only called, if the new last leaf was not existing yet
@ -169,20 +171,23 @@ void DataTree::resizeNumBytes(uint64_t newNumBytes) {
ASSERT(neededChildrenForRightBorderNode <= node->numChildren(), "Node has too few children");
// All children to the right of the new right-border-node are removed including their subtree.
while(node->numChildren() > neededChildrenForRightBorderNode) {
_nodeStore->removeSubtree(node->LastChild()->key());
_nodeStore->removeSubtree(node->depth()-1, node->LastChild()->key());
node->removeLastChild();
}
};
_traverseLeaves(newNumLeaves - 1, newNumLeaves, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
_numLeavesCache = newNumLeaves;
ASSERT(newNumBytes == _numStoredBytes(), "We resized to the wrong number of bytes ("+std::to_string(_numStoredBytes())+" instead of "+std::to_string(newNumBytes)+")");
}
uint64_t DataTree::maxBytesPerLeaf() const {
return _nodeStore->layout().maxBytesPerLeaf();
}
uint8_t DataTree::depth() const {
return _rootNode->depth();
}
}
}
}

2
src/blobstore/implementations/onblocks/datatreestore/DataTree.h
View File

@ -36,6 +36,8 @@ public:
uint32_t numLeaves() const;
uint64_t numStoredBytes() const;
uint8_t depth() const;
// only used by test cases
uint32_t _forceComputeNumLeaves() const;

8
src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.cpp
View File

@ -36,13 +36,13 @@ unique_ref<DataTree> DataTreeStore::createNewTree() {
}
void DataTreeStore::remove(unique_ref<DataTree> tree) {
auto key = tree->key();
cpputils::destruct(std::move(tree));
remove(key);
_nodeStore->removeSubtree(tree->releaseRootNode());
}
void DataTreeStore::remove(const blockstore::Key &key) {
_nodeStore->removeSubtree(key);
auto tree = load(key);
ASSERT(tree != none, "Tree to remove not found");
remove(std::move(*tree));
}
}

209
test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_Performance.cpp
View File

@ -19,6 +19,7 @@ public:
}
uint64_t maxChildrenPerInnerNode = nodeStore->layout().maxChildrenPerInnerNode();
uint64_t maxBytesPerLeaf = nodeStore->layout().maxBytesPerLeaf();
};
TEST_F(DataTreeTest_Performance, DeletingDoesntLoadLeaves_Twolevel_DeleteByTree) {
@ -28,7 +29,7 @@ TEST_F(DataTreeTest_Performance, DeletingDoesntLoadLeaves_Twolevel_DeleteByTree)
treeStore.remove(std::move(tree));
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // First loading is from loading the tree, second one from removing it (i.e. loading the root)
EXPECT_EQ(0u, blockStore->loadedBlocks().size());
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
@ -55,7 +56,7 @@ TEST_F(DataTreeTest_Performance, DeletingDoesntLoadLeaves_Threelevel_DeleteByTre
treeStore.remove(std::move(tree));
EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size());
EXPECT_EQ(maxChildrenPerInnerNode, blockStore->loadedBlocks().size());
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(1u + maxChildrenPerInnerNode + maxChildrenPerInnerNode*maxChildrenPerInnerNode, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
@ -223,7 +224,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInOldDe
Traverse(tree.get(), 4, maxChildrenPerInnerNode+2);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
EXPECT_EQ(2 + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // Add children to existing inner node
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
@ -237,7 +238,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInOldDe
Traverse(tree.get(), 4, maxChildrenPerInnerNode+2);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
EXPECT_EQ(2 + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(2u, blockStore->distinctWrittenBlocks().size()); // Resize last leaf and add children to existing inner node
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
@ -251,7 +252,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInNewDe
Traverse(tree.get(), maxChildrenPerInnerNode, maxChildrenPerInnerNode+2);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
EXPECT_EQ(2 + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // Add children to existing inner node
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
@ -265,8 +266,204 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInNewDe
Traverse(tree.get(), maxChildrenPerInnerNode, maxChildrenPerInnerNode+2);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
EXPECT_EQ(2 + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(2u, blockStore->distinctWrittenBlocks().size()); // Resize last leaf and add children to existing inner node
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_ZeroToZero) {
auto key = CreateLeafWithSize(0)->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(0);
EXPECT_EQ(0u, blockStore->loadedBlocks().size());
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_GrowOneLeaf) {
auto key = CreateLeafWithSize(0)->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(5);
EXPECT_EQ(0u, blockStore->loadedBlocks().size());
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_ShrinkOneLeaf) {
auto key = CreateLeafWithSize(5)->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(2);
EXPECT_EQ(0u, blockStore->loadedBlocks().size());
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_ShrinkOneLeafToZero) {
auto key = CreateLeafWithSize(5)->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(0);
EXPECT_EQ(0u, blockStore->loadedBlocks().size());
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_GrowOneLeafInLargerTree) {
auto key = CreateInner({CreateFullTwoLevel(), CreateInner({CreateLeaf(), CreateLeafWithSize(5)})})->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(maxBytesPerLeaf*(maxChildrenPerInnerNode+1)+6); // Grow by one byte
EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Load inner node and leaf
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_GrowByOneLeaf) {
auto key = CreateInner({CreateLeaf(), CreateLeaf()})->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(maxBytesPerLeaf*2+1); // Grow by one byte
EXPECT_EQ(1u, blockStore->loadedBlocks().size());
EXPECT_EQ(1u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // add child to inner node
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_GrowByOneLeaf_GrowLastLeaf) {
auto key = CreateInner({CreateLeaf(), CreateLeafWithSize(5)})->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(maxBytesPerLeaf*2+1); // Grow by one byte
EXPECT_EQ(1u, blockStore->loadedBlocks().size());
EXPECT_EQ(1u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(2u, blockStore->distinctWrittenBlocks().size()); // add child to inner node and resize old last leaf
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_ShrinkByOneLeaf) {
auto key = CreateInner({CreateLeaf(), CreateLeaf(), CreateLeaf()})->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(2*maxBytesPerLeaf-1);
EXPECT_EQ(1u, blockStore->loadedBlocks().size());
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(1u, blockStore->removedBlocks().size());
EXPECT_EQ(2u, blockStore->distinctWrittenBlocks().size()); // resize new last leaf and remove leaf from inner node
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_IncreaseTreeDepth_0to1) {
auto key = CreateLeaf()->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(maxBytesPerLeaf+1);
EXPECT_EQ(0u, blockStore->loadedBlocks().size());
EXPECT_EQ(2u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // rewrite root node to be an inner node
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_IncreaseTreeDepth_1to2) {
auto key = CreateFullTwoLevel()->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(maxBytesPerLeaf*maxChildrenPerInnerNode+1);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // check whether we have to grow last leaf
EXPECT_EQ(3u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // rewrite root node to be an inner node
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_IncreaseTreeDepth_0to2) {
auto key = CreateLeaf()->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(maxBytesPerLeaf*maxChildrenPerInnerNode+1);
EXPECT_EQ(0u, blockStore->loadedBlocks().size());
EXPECT_EQ(3u + maxChildrenPerInnerNode, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // rewrite root node to be an inner node
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_DecreaseTreeDepth_1to0) {
auto key = CreateInner({CreateLeaf(), CreateLeaf()})->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(maxBytesPerLeaf);
EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // read content of first leaf and load first leaf to replace root with it
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(2u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // rewrite root node to be a leaf
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_DecreaseTreeDepth_2to1) {
auto key = CreateInner({CreateFullTwoLevel(), CreateInner({CreateLeaf()})})->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(maxBytesPerLeaf*maxChildrenPerInnerNode);
EXPECT_EQ(4u, blockStore->loadedBlocks().size()); // load new last leaf (+inner node), load second inner node to remove its subtree, then load first child of root to replace root with its child.
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(3u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // rewrite root node to be a leaf
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, ResizeNumBytes_DecreaseTreeDepth_2to0) {
auto key = CreateInner({CreateFullTwoLevel(), CreateInner({CreateLeaf()})})->key();
auto tree = treeStore.load(key).value();
blockStore->resetCounters();
tree->resizeNumBytes(maxBytesPerLeaf);
EXPECT_EQ(5u, blockStore->loadedBlocks().size()); // load new last leaf (+inner node), load second inner node to remove its subtree, then 2x load first child of root to replace root with its child.
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(3u + maxChildrenPerInnerNode, blockStore->removedBlocks().size());
EXPECT_EQ(2u, blockStore->distinctWrittenBlocks().size()); // 2x rewrite root node to be a leaf
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}