hardcoresushi/libcryfs
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Traversal doesn't load leaves, but returns the key. This way, the write() call can (after a future commit) write to full leaves without loading them beforehand.

Browse Source
This commit is contained in:
Sebastian Messmer 2016-07-15 20:12:24 +02:00
parent 582917c1f5
commit d80b95514e
12 changed files with 161 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/blobstore/CMakeLists.txt
View File

@ -12,6 +12,7 @@ set(SOURCES
implementations/onblocks/datanodestore/DataNodeStore.cpp
implementations/onblocks/datatreestore/impl/algorithms.cpp
implementations/onblocks/datatreestore/impl/LeafTraverser.cpp
implementations/onblocks/datatreestore/LeafHandle.cpp
implementations/onblocks/datatreestore/DataTree.cpp
implementations/onblocks/datatreestore/DataTreeStore.cpp
implementations/onblocks/BlobOnBlocks.cpp

5
src/blobstore/implementations/onblocks/BlobOnBlocks.cpp
View File

@ -5,6 +5,7 @@
#include "utils/Math.h"
#include <cmath>
#include <cpp-utils/assert/assert.h>
#include "datatreestore/LeafHandle.h"
using std::function;
using std::unique_lock;
@ -14,6 +15,7 @@ using cpputils::Data;
using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataNodeLayout;
using blockstore::Key;
using blobstore::onblocks::datatreestore::LeafHandle;
namespace blobstore {
namespace onblocks {
@ -46,7 +48,8 @@ void BlobOnBlocks::_traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, funct
uint32_t firstLeaf = beginByte / maxBytesPerLeaf;
uint32_t endLeaf = utils::ceilDivision(endByte, maxBytesPerLeaf);
bool blobIsGrowingFromThisTraversal = false;
auto _onExistingLeaf = [&onExistingLeaf, beginByte, endByte, endLeaf, maxBytesPerLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex, DataLeafNode *leaf) {
auto _onExistingLeaf = [&onExistingLeaf, beginByte, endByte, endLeaf, maxBytesPerLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex, LeafHandle leafHandle) {
auto leaf = leafHandle.node();
uint64_t indexOfFirstLeafByte = leafIndex * maxBytesPerLeaf;
ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
ASSERT(leafIndex == endLeaf-1 || leaf->numBytes() == maxBytesPerLeaf, "All leafes but the rightmost one have to have maximal size");

7
src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp
View File

@ -96,7 +96,7 @@ uint32_t DataTree::_computeNumLeaves(const DataNode &node) const {
return numLeavesInLeftChildren + numLeavesInRightChild;
}
void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, DataLeafNode* leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf) {
void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf) {
//TODO Can we allow multiple runs of traverseLeaves() in parallel? Also in parallel with resizeNumBytes()?
std::unique_lock<shared_mutex> lock(_mutex);
ASSERT(beginIndex <= endIndex, "Invalid parameters");
@ -111,7 +111,7 @@ void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<v
}
void DataTree::_traverseLeaves(uint32_t beginIndex, uint32_t endIndex,
function<void (uint32_t index, DataLeafNode* leaf)> onExistingLeaf,
function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf,
function<Data (uint32_t index)> onCreateLeaf,
function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
_rootNode = LeafTraverser(_nodeStore).traverseAndReturnRoot(std::move(_rootNode), beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
@ -151,7 +151,8 @@ void DataTree::resizeNumBytes(uint64_t newNumBytes) {
uint32_t newNumLeaves = std::max(UINT64_C(1), utils::ceilDivision(newNumBytes, _nodeStore->layout().maxBytesPerLeaf()));
uint32_t newLastLeafSize = newNumBytes - (newNumLeaves-1) * _nodeStore->layout().maxBytesPerLeaf();
uint32_t maxChildrenPerInnerNode = _nodeStore->layout().maxChildrenPerInnerNode();
auto onExistingLeaf = [newLastLeafSize] (uint32_t /*index*/, datanodestore::DataLeafNode* leaf) {
auto onExistingLeaf = [newLastLeafSize] (uint32_t /*index*/, LeafHandle leafHandle) {
auto leaf = leafHandle.node();
// This is only called, if the new last leaf was already existing
if (leaf->numBytes() != newLastLeafSize) {
leaf->resize(newLastLeafSize);

5
src/blobstore/implementations/onblocks/datatreestore/DataTree.h
View File

@ -9,6 +9,7 @@
//TODO Replace with C++14 once std::shared_mutex is supported
#include <boost/thread/shared_mutex.hpp>
#include <blockstore/utils/Key.h>
#include "LeafHandle.h"
namespace blobstore {
namespace onblocks {
@ -30,7 +31,7 @@ public:
//Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
uint64_t maxBytesPerLeaf() const;
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, datanodestore::DataLeafNode* leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf);
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf);
void resizeNumBytes(uint64_t newNumBytes);
uint32_t numLeaves() const;
@ -55,7 +56,7 @@ private:
//TODO Use underscore for private methods
void _traverseLeaves(uint32_t beginIndex, uint32_t endIndex,
std::function<void (uint32_t index, datanodestore::DataLeafNode* leaf)> onExistingLeaf,
std::function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
uint32_t leavesPerFullChild(const datanodestore::DataInnerNode &root) const;

1
src/blobstore/implementations/onblocks/datatreestore/LeafHandle.cpp Normal file
View File

@ -0,0 +1 @@
#include "LeafHandle.h"

56
src/blobstore/implementations/onblocks/datatreestore/LeafHandle.h Normal file
View File

@ -0,0 +1,56 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_LEAFHANDLE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_LEAFHANDLE_H_
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/optional_ownership_ptr.h>
#include "blobstore/implementations/onblocks/datanodestore/DataNodeStore.h"
#include "blobstore/implementations/onblocks/datanodestore/DataLeafNode.h"
namespace blobstore {
namespace onblocks {
namespace datatreestore {
class LeafHandle final {
public:
LeafHandle(datanodestore::DataNodeStore *nodeStore, const blockstore::Key &key)
:_nodeStore(nodeStore), _key(key), _leaf(cpputils::null<datanodestore::DataLeafNode>()) {
}
LeafHandle(datanodestore::DataNodeStore *nodeStore, datanodestore::DataLeafNode *node)
: _nodeStore(nodeStore), _key(node->key()), _leaf(cpputils::WithoutOwnership<datanodestore::DataLeafNode>(node)) {
}
LeafHandle(LeafHandle &&rhs) = default;
const blockstore::Key &key() {
return _key;
}
datanodestore::DataLeafNode *node() {
if (_leaf.get() == nullptr) {
auto loaded = _nodeStore->load(_key);
ASSERT(loaded != boost::none, "Leaf not found");
auto leaf = cpputils::dynamic_pointer_move<datanodestore::DataLeafNode>(*loaded);
ASSERT(leaf != boost::none, "Loaded leaf is not leaf node");
_leaf = cpputils::WithOwnership(std::move(*leaf));
}
return _leaf.get();
}
private:
datanodestore::DataNodeStore *_nodeStore;
blockstore::Key _key;
cpputils::optional_ownership_ptr<datanodestore::DataLeafNode> _leaf;
DISALLOW_COPY_AND_ASSIGN(LeafHandle);
};
}
}
}
#endif

93
src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.cpp
View File

@ -24,11 +24,11 @@ namespace blobstore {
: _nodeStore(nodeStore) {
}
unique_ref<DataNode> LeafTraverser::traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, DataLeafNode* leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
unique_ref<DataNode> LeafTraverser::traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
return _traverseAndReturnRoot(std::move(root), beginIndex, endIndex, true, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
unique_ref<DataNode> LeafTraverser::_traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, function<void (uint32_t index, DataLeafNode* leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
unique_ref<DataNode> LeafTraverser::_traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
ASSERT(beginIndex <= endIndex, "Invalid parameters");
//TODO Test cases with numLeaves < / >= beginIndex, ideally test all configurations:
@ -39,7 +39,23 @@ namespace blobstore {
uint32_t maxLeavesForDepth = _maxLeavesForTreeDepth(root->depth());
bool increaseTreeDepth = endIndex > maxLeavesForDepth;
_traverseExistingSubtree(root.get(), std::min(beginIndex, maxLeavesForDepth), std::min(endIndex, maxLeavesForDepth), 0, isLeftBorderOfTraversal, increaseTreeDepth, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
if (root->depth() == 0) {
DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root.get());
ASSERT(leaf != nullptr, "Depth 0 has to be leaf node");
if (increaseTreeDepth && leaf->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) {
leaf->resize(_nodeStore->layout().maxBytesPerLeaf());
}
if (beginIndex == 0 && endIndex == 1) {
onExistingLeaf(0, LeafHandle(_nodeStore, leaf));
}
} else {
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root.get());
ASSERT(inner != nullptr, "Depth != 0 has to be leaf node");
_traverseExistingSubtree(inner, std::min(beginIndex, maxLeavesForDepth),
std::min(endIndex, maxLeavesForDepth), 0, isLeftBorderOfTraversal,
increaseTreeDepth, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
// If the traversal goes too far right for a tree this depth, increase tree depth by one and continue traversal.
// This is recursive, i.e. will be repeated if the tree is still not deep enough.
@ -61,30 +77,43 @@ namespace blobstore {
return DataNode::convertToNewInnerNode(std::move(root), _nodeStore->layout(), *copyOfOldRoot);
}
void LeafTraverser::_traverseExistingSubtree(DataNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf, function<void (uint32_t index, DataLeafNode* leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
void LeafTraverser::_traverseExistingSubtree(const blockstore::Key &key, uint8_t depth, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf, function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
if (depth == 0) {
ASSERT(beginIndex <= 1 && endIndex <= 1,
"If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1.");
LeafHandle leafHandle(_nodeStore, key);
if (growLastLeaf) {
if (leafHandle.node()->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) {
leafHandle.node()->resize(_nodeStore->layout().maxBytesPerLeaf());
}
}
if (beginIndex == 0 && endIndex == 1) {
onExistingLeaf(leafOffset, std::move(leafHandle));
}
} else {
auto node = _nodeStore->load(key);
if (node == none) {
throw std::runtime_error("Couldn't find child node " + key.ToString());
}
auto inner = dynamic_pointer_move<DataInnerNode>(*node);
ASSERT(inner != none, "Has to be either leaf or inner node");
ASSERT((*inner)->depth() == depth, "Wrong depth given");
_traverseExistingSubtree(inner->get(), beginIndex, endIndex, leafOffset, isLeftBorderOfTraversal,
growLastLeaf, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
}
void LeafTraverser::_traverseExistingSubtree(DataInnerNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf, function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
ASSERT(beginIndex <= endIndex, "Invalid parameters");
//TODO Call callbacks for different leaves in parallel.
DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root);
if (leaf != nullptr) {
ASSERT(beginIndex <= 1 && endIndex <= 1, "If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1.");
if (growLastLeaf && leaf->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) {
leaf->resize(_nodeStore->layout().maxBytesPerLeaf());
}
if (beginIndex == 0 && endIndex == 1) {
onExistingLeaf(leafOffset, leaf);
}
return;
}
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root);
uint32_t leavesPerChild = _maxLeavesForTreeDepth(inner->depth()-1);
uint32_t leavesPerChild = _maxLeavesForTreeDepth(root->depth()-1);
uint32_t beginChild = beginIndex/leavesPerChild;
uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild);
ASSERT(endChild <= _nodeStore->layout().maxChildrenPerInnerNode(), "Traversal region would need increasing the tree depth. This should have happened before calling this function.");
uint32_t numChildren = inner->numChildren();
uint32_t numChildren = root->numChildren();
ASSERT(!growLastLeaf || endChild >= numChildren, "Can only grow last leaf if it exists");
bool shouldGrowLastExistingLeaf = growLastLeaf || endChild > numChildren;
@ -92,32 +121,24 @@ namespace blobstore {
// we still have to descend to the last old child to fill it with leaves and grow the last old leaf.
if (isLeftBorderOfTraversal && beginChild >= numChildren) {
ASSERT(numChildren > 0, "Node doesn't have children.");
auto childKey = inner->getChild(numChildren-1)->key();
auto childNode = _nodeStore->load(childKey);
if (childNode == none) {
throw std::runtime_error("Couldn't find child node "+childKey.ToString());
}
auto childKey = root->getChild(numChildren-1)->key();
uint32_t childOffset = (numChildren-1) * leavesPerChild;
_traverseExistingSubtree(childNode->get(), leavesPerChild, leavesPerChild, childOffset, true, true,
[] (uint32_t /*index*/, DataLeafNode* /*leaf*/) {ASSERT(false, "We don't actually traverse any leaves.");},
_traverseExistingSubtree(childKey, root->depth()-1, leavesPerChild, leavesPerChild, childOffset, true, true,
[] (uint32_t /*index*/, LeafHandle /*leaf*/) {ASSERT(false, "We don't actually traverse any leaves.");},
[] (uint32_t /*index*/) -> Data {ASSERT(false, "We don't actually traverse any leaves.");},
[] (DataInnerNode* /*node*/) {ASSERT(false, "We don't actually traverse any leaves.");});
}
// Traverse existing children
for (uint32_t childIndex = beginChild; childIndex < std::min(endChild, numChildren); ++childIndex) {
auto childKey = inner->getChild(childIndex)->key();
auto childNode = _nodeStore->load(childKey);
if (childNode == none) {
throw std::runtime_error("Couldn't find child node "+childKey.ToString());
}
auto childKey = root->getChild(childIndex)->key();
uint32_t childOffset = childIndex * leavesPerChild;
uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset);
uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
bool isFirstChild = (childIndex == beginChild);
bool isLastChild = (childIndex == numChildren - 1);
ASSERT(localEndIndex <= leavesPerChild, "We don't want the child to add a tree level because it doesn't have enough space for the traversal.");
_traverseExistingSubtree(childNode->get(), localBeginIndex, localEndIndex, leafOffset + childOffset, isLeftBorderOfTraversal && isFirstChild,
_traverseExistingSubtree(childKey, root->depth()-1, localBeginIndex, localEndIndex, leafOffset + childOffset, isLeftBorderOfTraversal && isFirstChild,
shouldGrowLastExistingLeaf && isLastChild, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
@ -127,13 +148,13 @@ namespace blobstore {
uint32_t localBeginIndex = std::min(leavesPerChild, utils::maxZeroSubtraction(beginIndex, childOffset));
uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
auto leafCreator = (childIndex >= beginChild) ? onCreateLeaf : _createMaxSizeLeaf();
auto child = _createNewSubtree(localBeginIndex, localEndIndex, leafOffset + childOffset, inner->depth() - 1, leafCreator, onBacktrackFromSubtree);
inner->addChild(*child);
auto child = _createNewSubtree(localBeginIndex, localEndIndex, leafOffset + childOffset, root->depth() - 1, leafCreator, onBacktrackFromSubtree);
root->addChild(*child);
}
// This is only a backtrack, if we actually visited a leaf here.
if (endIndex > beginIndex) {
onBacktrackFromSubtree(inner);
onBacktrackFromSubtree(root);
}
}

25
src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.h
View File

@ -6,6 +6,7 @@
#include <cpp-utils/pointer/unique_ref.h>
#include <cpp-utils/data/Data.h>
#include <blockstore/utils/Key.h>
#include "blobstore/implementations/onblocks/datatreestore/LeafHandle.h"
namespace blobstore {
namespace onblocks {
@ -27,21 +28,25 @@ namespace blobstore {
LeafTraverser(datanodestore::DataNodeStore *nodeStore);
cpputils::unique_ref<datanodestore::DataNode> traverseAndReturnRoot(
cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex,
std::function<void (uint32_t index, datanodestore::DataLeafNode* leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex,
std::function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
private:
datanodestore::DataNodeStore *_nodeStore;
cpputils::unique_ref<datanodestore::DataNode> _traverseAndReturnRoot(
cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal,
std::function<void (uint32_t index, datanodestore::DataLeafNode* leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
void _traverseExistingSubtree(datanodestore::DataNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf,
std::function<void (uint32_t index, datanodestore::DataLeafNode* leaf)> onExistingLeaf,
cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal,
std::function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
void _traverseExistingSubtree(datanodestore::DataInnerNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf,
std::function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
void _traverseExistingSubtree(const blockstore::Key &key, uint8_t depth, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf,
std::function<void (uint32_t index, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
cpputils::unique_ref<datanodestore::DataInnerNode> _increaseTreeDepth(cpputils::unique_ref<datanodestore::DataNode> root);

3
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h
View File

@ -4,6 +4,7 @@
#include <parallelaccessstore/ParallelAccessStore.h>
#include "../datatreestore/DataTree.h"
#include "blobstore/implementations/onblocks/datatreestore/LeafHandle.h"
namespace blobstore {
namespace onblocks {
@ -21,7 +22,7 @@ public:
return _baseTree->maxBytesPerLeaf();
}
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, datanodestore::DataLeafNode* leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf) {
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, datatreestore::LeafHandle leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf) {
return _baseTree->traverseLeaves(beginIndex, endIndex, onExistingLeaf, onCreateLeaf);
}

17
test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_Performance.cpp
View File

@ -7,6 +7,7 @@ using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datatreestore::DataTree;
using blobstore::onblocks::datatreestore::LeafHandle;
using blockstore::Key;
using blockstore::testfake::FakeBlockStore;
using cpputils::Data;
@ -15,7 +16,7 @@ using cpputils::make_unique_ref;
class DataTreeTest_Performance: public DataTreeTest {
public:
void Traverse(DataTree *tree, uint64_t beginIndex, uint64_t endIndex) {
tree->traverseLeaves(beginIndex, endIndex, [] (uint32_t /*index*/, DataLeafNode* /*leaf*/) {}, [this] (uint32_t /*index*/) -> Data {return Data(maxChildrenPerInnerNode).FillWithZeroes();});
tree->traverseLeaves(beginIndex, endIndex, [] (uint32_t /*index*/, LeafHandle /*leaf*/) {}, [this] (uint32_t /*index*/) -> Data {return Data(maxChildrenPerInnerNode).FillWithZeroes();});
}
uint64_t maxChildrenPerInnerNode = nodeStore->layout().maxChildrenPerInnerNode();
@ -83,7 +84,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_All) {
Traverse(tree.get(), 0, maxChildrenPerInnerNode);
EXPECT_EQ(maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads all leaves (not the root, because it is already loaded in the tree)
EXPECT_EQ(0u, blockStore->loadedBlocks().size()); // Doesn't actually load the leaves, but returns the keys of the leaves to the callback
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
@ -97,7 +98,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_Some) {
Traverse(tree.get(), 3, 5);
EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads both leaves (not the root, because it is already loaded in the tree)
EXPECT_EQ(0u, blockStore->loadedBlocks().size()); // Doesn't actually load the leaves, but returns the keys of the leaves to the callback
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
@ -111,7 +112,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_All) {
Traverse(tree.get(), 0, maxChildrenPerInnerNode * maxChildrenPerInnerNode);
EXPECT_EQ(maxChildrenPerInnerNode + maxChildrenPerInnerNode * maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner nodes and all leaves once (not the root, because it is already loaded in the tree)
EXPECT_EQ(maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner nodes. Doesn't load the leaves, but returns the keys of the leaves to the callback
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
@ -125,7 +126,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InOneInner) {
Traverse(tree.get(), 3, 5);
EXPECT_EQ(3u, blockStore->loadedBlocks().size()); // Loads inner node and both leaves (not the root, because it is already loaded in the tree)
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, but returns the keys of the leaves to the callback
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
@ -139,7 +140,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InTwoInner) {
Traverse(tree.get(), 3, 3 + maxChildrenPerInnerNode);
EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner node and both leaves (not the root, because it is already loaded in the tree)
EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads both inner node
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
@ -153,7 +154,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_WholeInner) {
Traverse(tree.get(), maxChildrenPerInnerNode, 2*maxChildrenPerInnerNode);
EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner node and leaves (not the root, because it is already loaded in the tree)f
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, but returns the keys of the leaves to the callback
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
@ -209,7 +210,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTree_StartingAtBeginOfChi
Traverse(tree.get(), maxChildrenPerInnerNode, 3*maxChildrenPerInnerNode);
EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Inner node and its leaves
EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads inner node and one leaf to check whether we have to grow it. Doesn't load the leaves, but returns the keys of the leaves to the callback.
EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // Creates an inner node and its leaves
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // add children to existing inner node

3
test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_ResizeByTraversing.cpp
View File

@ -19,6 +19,7 @@ using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datanodestore::DataNodeLayout;
using blobstore::onblocks::datatreestore::DataTree;
using blobstore::onblocks::datatreestore::LeafHandle;
using blobstore::onblocks::utils::ceilDivision;
using blockstore::Key;
using cpputils::Data;
@ -110,7 +111,7 @@ public:
void GrowTree(DataTree *tree) {
uint64_t maxBytesPerLeaf = tree->maxBytesPerLeaf();
tree->traverseLeaves(traversalBeginIndex, newNumberOfLeaves, [] (uint32_t, DataLeafNode*){}, [maxBytesPerLeaf] (uint32_t) -> Data { return Data(maxBytesPerLeaf).FillWithZeroes();});
tree->traverseLeaves(traversalBeginIndex, newNumberOfLeaves, [] (uint32_t, LeafHandle){}, [maxBytesPerLeaf] (uint32_t) -> Data { return Data(maxBytesPerLeaf).FillWithZeroes();});
tree->flush();
}

13
test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_TraverseLeaves.cpp
View File

@ -7,6 +7,7 @@ using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datatreestore::DataTree;
using blobstore::onblocks::datatreestore::LeafHandle;
using blockstore::Key;
using cpputils::unique_ref;
@ -63,8 +64,8 @@ public:
void TraverseLeaves(DataNode *root, uint32_t beginIndex, uint32_t endIndex) {
root->flush();
auto tree = treeStore.load(root->key()).value();
tree->traverseLeaves(beginIndex, endIndex, [this] (uint32_t nodeIndex, DataLeafNode *leaf) {
traversor.calledExistingLeaf(leaf, nodeIndex);
tree->traverseLeaves(beginIndex, endIndex, [this] (uint32_t nodeIndex, LeafHandle leaf) {
traversor.calledExistingLeaf(leaf.node(), nodeIndex);
}, [this] (uint32_t nodeIndex) -> Data {
return traversor.calledCreateLeaf(nodeIndex)->copy();
});
@ -393,9 +394,9 @@ TEST_F(DataTreeTest_TraverseLeaves, LastLeafIsAlreadyResizedInCallback) {
auto root = CreateLeaf();
root->flush();
auto tree = treeStore.load(root->key()).value();
tree->traverseLeaves(0, 2, [this] (uint32_t leafIndex, DataLeafNode *leaf) {
tree->traverseLeaves(0, 2, [this] (uint32_t leafIndex, LeafHandle leaf) {
if (leafIndex == 0) {
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf->numBytes());
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf.node()->numBytes());
} else {
EXPECT_TRUE(false) << "only two nodes";
}
@ -408,8 +409,8 @@ TEST_F(DataTreeTest_TraverseLeaves, LastLeafIsAlreadyResizedInCallback_TwoLevel)
auto root = CreateFullTwoLevelWithLastLeafSize(5);
root->flush();
auto tree = treeStore.load(root->key()).value();
tree->traverseLeaves(0, nodeStore->layout().maxChildrenPerInnerNode()+1, [this] (uint32_t /*leafIndex*/, DataLeafNode *leaf) {
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf->numBytes());
tree->traverseLeaves(0, nodeStore->layout().maxChildrenPerInnerNode()+1, [this] (uint32_t /*leafIndex*/, LeafHandle leaf) {
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf.node()->numBytes());
}, [this] (uint32_t /*nodeIndex*/) -> Data {
return Data(1);
});