diff --git a/src/blobstore/CMakeLists.txt b/src/blobstore/CMakeLists.txt index f807d367..9bf68976 100644 --- a/src/blobstore/CMakeLists.txt +++ b/src/blobstore/CMakeLists.txt @@ -12,6 +12,7 @@ set(SOURCES implementations/onblocks/datanodestore/DataNodeStore.cpp implementations/onblocks/datatreestore/impl/algorithms.cpp implementations/onblocks/datatreestore/impl/LeafTraverser.cpp + implementations/onblocks/datatreestore/LeafHandle.cpp implementations/onblocks/datatreestore/DataTree.cpp implementations/onblocks/datatreestore/DataTreeStore.cpp implementations/onblocks/BlobOnBlocks.cpp diff --git a/src/blobstore/implementations/onblocks/BlobOnBlocks.cpp b/src/blobstore/implementations/onblocks/BlobOnBlocks.cpp index 3ffcca61..5fc026b6 100644 --- a/src/blobstore/implementations/onblocks/BlobOnBlocks.cpp +++ b/src/blobstore/implementations/onblocks/BlobOnBlocks.cpp @@ -5,6 +5,7 @@ #include "utils/Math.h" #include #include +#include "datatreestore/LeafHandle.h" using std::function; using std::unique_lock; @@ -14,6 +15,7 @@ using cpputils::Data; using blobstore::onblocks::datanodestore::DataLeafNode; using blobstore::onblocks::datanodestore::DataNodeLayout; using blockstore::Key; +using blobstore::onblocks::datatreestore::LeafHandle; namespace blobstore { namespace onblocks { @@ -46,7 +48,8 @@ void BlobOnBlocks::_traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, funct uint32_t firstLeaf = beginByte / maxBytesPerLeaf; uint32_t endLeaf = utils::ceilDivision(endByte, maxBytesPerLeaf); bool blobIsGrowingFromThisTraversal = false; - auto _onExistingLeaf = [&onExistingLeaf, beginByte, endByte, endLeaf, maxBytesPerLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex, DataLeafNode *leaf) { + auto _onExistingLeaf = [&onExistingLeaf, beginByte, endByte, endLeaf, maxBytesPerLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex, LeafHandle leafHandle) { + auto leaf = leafHandle.node(); uint64_t indexOfFirstLeafByte = leafIndex * maxBytesPerLeaf; ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right"); ASSERT(leafIndex == endLeaf-1 || leaf->numBytes() == maxBytesPerLeaf, "All leafes but the rightmost one have to have maximal size"); diff --git a/src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp b/src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp index 4a5a62e2..c2338b65 100644 --- a/src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp +++ b/src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp @@ -96,7 +96,7 @@ uint32_t DataTree::_computeNumLeaves(const DataNode &node) const { return numLeavesInLeftChildren + numLeavesInRightChild; } -void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function onExistingLeaf, function onCreateLeaf) { +void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function onExistingLeaf, function onCreateLeaf) { //TODO Can we allow multiple runs of traverseLeaves() in parallel? Also in parallel with resizeNumBytes()? std::unique_lock lock(_mutex); ASSERT(beginIndex <= endIndex, "Invalid parameters"); @@ -111,7 +111,7 @@ void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function onExistingLeaf, + function onExistingLeaf, function onCreateLeaf, function onBacktrackFromSubtree) { _rootNode = LeafTraverser(_nodeStore).traverseAndReturnRoot(std::move(_rootNode), beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree); @@ -151,7 +151,8 @@ void DataTree::resizeNumBytes(uint64_t newNumBytes) { uint32_t newNumLeaves = std::max(UINT64_C(1), utils::ceilDivision(newNumBytes, _nodeStore->layout().maxBytesPerLeaf())); uint32_t newLastLeafSize = newNumBytes - (newNumLeaves-1) * _nodeStore->layout().maxBytesPerLeaf(); uint32_t maxChildrenPerInnerNode = _nodeStore->layout().maxChildrenPerInnerNode(); - auto onExistingLeaf = [newLastLeafSize] (uint32_t /*index*/, datanodestore::DataLeafNode* leaf) { + auto onExistingLeaf = [newLastLeafSize] (uint32_t /*index*/, LeafHandle leafHandle) { + auto leaf = leafHandle.node(); // This is only called, if the new last leaf was already existing if (leaf->numBytes() != newLastLeafSize) { leaf->resize(newLastLeafSize); diff --git a/src/blobstore/implementations/onblocks/datatreestore/DataTree.h b/src/blobstore/implementations/onblocks/datatreestore/DataTree.h index dc538eb9..2b362e8d 100644 --- a/src/blobstore/implementations/onblocks/datatreestore/DataTree.h +++ b/src/blobstore/implementations/onblocks/datatreestore/DataTree.h @@ -9,6 +9,7 @@ //TODO Replace with C++14 once std::shared_mutex is supported #include #include +#include "LeafHandle.h" namespace blobstore { namespace onblocks { @@ -30,7 +31,7 @@ public: //Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs. uint64_t maxBytesPerLeaf() const; - void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function onExistingLeaf, std::function onCreateLeaf); + void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function onExistingLeaf, std::function onCreateLeaf); void resizeNumBytes(uint64_t newNumBytes); uint32_t numLeaves() const; @@ -55,7 +56,7 @@ private: //TODO Use underscore for private methods void _traverseLeaves(uint32_t beginIndex, uint32_t endIndex, - std::function onExistingLeaf, + std::function onExistingLeaf, std::function onCreateLeaf, std::function onBacktrackFromSubtree); uint32_t leavesPerFullChild(const datanodestore::DataInnerNode &root) const; diff --git a/src/blobstore/implementations/onblocks/datatreestore/LeafHandle.cpp b/src/blobstore/implementations/onblocks/datatreestore/LeafHandle.cpp new file mode 100644 index 00000000..5e60864e --- /dev/null +++ b/src/blobstore/implementations/onblocks/datatreestore/LeafHandle.cpp @@ -0,0 +1 @@ +#include "LeafHandle.h" diff --git a/src/blobstore/implementations/onblocks/datatreestore/LeafHandle.h b/src/blobstore/implementations/onblocks/datatreestore/LeafHandle.h new file mode 100644 index 00000000..9ba88919 --- /dev/null +++ b/src/blobstore/implementations/onblocks/datatreestore/LeafHandle.h @@ -0,0 +1,56 @@ +#pragma once +#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_LEAFHANDLE_H_ +#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_LEAFHANDLE_H_ + +#include +#include +#include "blobstore/implementations/onblocks/datanodestore/DataNodeStore.h" +#include "blobstore/implementations/onblocks/datanodestore/DataLeafNode.h" + +namespace blobstore { + namespace onblocks { + namespace datatreestore { + + class LeafHandle final { + public: + LeafHandle(datanodestore::DataNodeStore *nodeStore, const blockstore::Key &key) + :_nodeStore(nodeStore), _key(key), _leaf(cpputils::null()) { + } + + LeafHandle(datanodestore::DataNodeStore *nodeStore, datanodestore::DataLeafNode *node) + : _nodeStore(nodeStore), _key(node->key()), _leaf(cpputils::WithoutOwnership(node)) { + } + + LeafHandle(LeafHandle &&rhs) = default; + + const blockstore::Key &key() { + return _key; + } + + datanodestore::DataLeafNode *node() { + if (_leaf.get() == nullptr) { + auto loaded = _nodeStore->load(_key); + ASSERT(loaded != boost::none, "Leaf not found"); + auto leaf = cpputils::dynamic_pointer_move(*loaded); + ASSERT(leaf != boost::none, "Loaded leaf is not leaf node"); + + _leaf = cpputils::WithOwnership(std::move(*leaf)); + } + + return _leaf.get(); + } + + private: + datanodestore::DataNodeStore *_nodeStore; + blockstore::Key _key; + cpputils::optional_ownership_ptr _leaf; + + DISALLOW_COPY_AND_ASSIGN(LeafHandle); + }; + + + } + } +} + +#endif diff --git a/src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.cpp b/src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.cpp index 0ec9ecfc..ab363ab0 100644 --- a/src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.cpp +++ b/src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.cpp @@ -24,11 +24,11 @@ namespace blobstore { : _nodeStore(nodeStore) { } - unique_ref LeafTraverser::traverseAndReturnRoot(unique_ref root, uint32_t beginIndex, uint32_t endIndex, function onExistingLeaf, function onCreateLeaf, function onBacktrackFromSubtree) { + unique_ref LeafTraverser::traverseAndReturnRoot(unique_ref root, uint32_t beginIndex, uint32_t endIndex, function onExistingLeaf, function onCreateLeaf, function onBacktrackFromSubtree) { return _traverseAndReturnRoot(std::move(root), beginIndex, endIndex, true, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree); } - unique_ref LeafTraverser::_traverseAndReturnRoot(unique_ref root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, function onExistingLeaf, function onCreateLeaf, function onBacktrackFromSubtree) { + unique_ref LeafTraverser::_traverseAndReturnRoot(unique_ref root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, function onExistingLeaf, function onCreateLeaf, function onBacktrackFromSubtree) { ASSERT(beginIndex <= endIndex, "Invalid parameters"); //TODO Test cases with numLeaves < / >= beginIndex, ideally test all configurations: @@ -39,7 +39,23 @@ namespace blobstore { uint32_t maxLeavesForDepth = _maxLeavesForTreeDepth(root->depth()); bool increaseTreeDepth = endIndex > maxLeavesForDepth; - _traverseExistingSubtree(root.get(), std::min(beginIndex, maxLeavesForDepth), std::min(endIndex, maxLeavesForDepth), 0, isLeftBorderOfTraversal, increaseTreeDepth, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree); + if (root->depth() == 0) { + DataLeafNode *leaf = dynamic_cast(root.get()); + ASSERT(leaf != nullptr, "Depth 0 has to be leaf node"); + + if (increaseTreeDepth && leaf->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) { + leaf->resize(_nodeStore->layout().maxBytesPerLeaf()); + } + if (beginIndex == 0 && endIndex == 1) { + onExistingLeaf(0, LeafHandle(_nodeStore, leaf)); + } + } else { + DataInnerNode *inner = dynamic_cast(root.get()); + ASSERT(inner != nullptr, "Depth != 0 has to be leaf node"); + _traverseExistingSubtree(inner, std::min(beginIndex, maxLeavesForDepth), + std::min(endIndex, maxLeavesForDepth), 0, isLeftBorderOfTraversal, + increaseTreeDepth, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree); + } // If the traversal goes too far right for a tree this depth, increase tree depth by one and continue traversal. // This is recursive, i.e. will be repeated if the tree is still not deep enough. @@ -61,30 +77,43 @@ namespace blobstore { return DataNode::convertToNewInnerNode(std::move(root), _nodeStore->layout(), *copyOfOldRoot); } - void LeafTraverser::_traverseExistingSubtree(DataNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf, function onExistingLeaf, function onCreateLeaf, function onBacktrackFromSubtree) { + void LeafTraverser::_traverseExistingSubtree(const blockstore::Key &key, uint8_t depth, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf, function onExistingLeaf, function onCreateLeaf, function onBacktrackFromSubtree) { + if (depth == 0) { + ASSERT(beginIndex <= 1 && endIndex <= 1, + "If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1."); + LeafHandle leafHandle(_nodeStore, key); + if (growLastLeaf) { + if (leafHandle.node()->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) { + leafHandle.node()->resize(_nodeStore->layout().maxBytesPerLeaf()); + } + } + if (beginIndex == 0 && endIndex == 1) { + onExistingLeaf(leafOffset, std::move(leafHandle)); + } + } else { + auto node = _nodeStore->load(key); + if (node == none) { + throw std::runtime_error("Couldn't find child node " + key.ToString()); + } + + auto inner = dynamic_pointer_move(*node); + ASSERT(inner != none, "Has to be either leaf or inner node"); + ASSERT((*inner)->depth() == depth, "Wrong depth given"); + _traverseExistingSubtree(inner->get(), beginIndex, endIndex, leafOffset, isLeftBorderOfTraversal, + growLastLeaf, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree); + } + } + + void LeafTraverser::_traverseExistingSubtree(DataInnerNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf, function onExistingLeaf, function onCreateLeaf, function onBacktrackFromSubtree) { ASSERT(beginIndex <= endIndex, "Invalid parameters"); //TODO Call callbacks for different leaves in parallel. - DataLeafNode *leaf = dynamic_cast(root); - if (leaf != nullptr) { - ASSERT(beginIndex <= 1 && endIndex <= 1, "If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1."); - if (growLastLeaf && leaf->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) { - leaf->resize(_nodeStore->layout().maxBytesPerLeaf()); - } - if (beginIndex == 0 && endIndex == 1) { - onExistingLeaf(leafOffset, leaf); - } - return; - } - - DataInnerNode *inner = dynamic_cast(root); - - uint32_t leavesPerChild = _maxLeavesForTreeDepth(inner->depth()-1); + uint32_t leavesPerChild = _maxLeavesForTreeDepth(root->depth()-1); uint32_t beginChild = beginIndex/leavesPerChild; uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild); ASSERT(endChild <= _nodeStore->layout().maxChildrenPerInnerNode(), "Traversal region would need increasing the tree depth. This should have happened before calling this function."); - uint32_t numChildren = inner->numChildren(); + uint32_t numChildren = root->numChildren(); ASSERT(!growLastLeaf || endChild >= numChildren, "Can only grow last leaf if it exists"); bool shouldGrowLastExistingLeaf = growLastLeaf || endChild > numChildren; @@ -92,32 +121,24 @@ namespace blobstore { // we still have to descend to the last old child to fill it with leaves and grow the last old leaf. if (isLeftBorderOfTraversal && beginChild >= numChildren) { ASSERT(numChildren > 0, "Node doesn't have children."); - auto childKey = inner->getChild(numChildren-1)->key(); - auto childNode = _nodeStore->load(childKey); - if (childNode == none) { - throw std::runtime_error("Couldn't find child node "+childKey.ToString()); - } + auto childKey = root->getChild(numChildren-1)->key(); uint32_t childOffset = (numChildren-1) * leavesPerChild; - _traverseExistingSubtree(childNode->get(), leavesPerChild, leavesPerChild, childOffset, true, true, - [] (uint32_t /*index*/, DataLeafNode* /*leaf*/) {ASSERT(false, "We don't actually traverse any leaves.");}, + _traverseExistingSubtree(childKey, root->depth()-1, leavesPerChild, leavesPerChild, childOffset, true, true, + [] (uint32_t /*index*/, LeafHandle /*leaf*/) {ASSERT(false, "We don't actually traverse any leaves.");}, [] (uint32_t /*index*/) -> Data {ASSERT(false, "We don't actually traverse any leaves.");}, [] (DataInnerNode* /*node*/) {ASSERT(false, "We don't actually traverse any leaves.");}); } // Traverse existing children for (uint32_t childIndex = beginChild; childIndex < std::min(endChild, numChildren); ++childIndex) { - auto childKey = inner->getChild(childIndex)->key(); - auto childNode = _nodeStore->load(childKey); - if (childNode == none) { - throw std::runtime_error("Couldn't find child node "+childKey.ToString()); - } + auto childKey = root->getChild(childIndex)->key(); uint32_t childOffset = childIndex * leavesPerChild; uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset); uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset); bool isFirstChild = (childIndex == beginChild); bool isLastChild = (childIndex == numChildren - 1); ASSERT(localEndIndex <= leavesPerChild, "We don't want the child to add a tree level because it doesn't have enough space for the traversal."); - _traverseExistingSubtree(childNode->get(), localBeginIndex, localEndIndex, leafOffset + childOffset, isLeftBorderOfTraversal && isFirstChild, + _traverseExistingSubtree(childKey, root->depth()-1, localBeginIndex, localEndIndex, leafOffset + childOffset, isLeftBorderOfTraversal && isFirstChild, shouldGrowLastExistingLeaf && isLastChild, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree); } @@ -127,13 +148,13 @@ namespace blobstore { uint32_t localBeginIndex = std::min(leavesPerChild, utils::maxZeroSubtraction(beginIndex, childOffset)); uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset); auto leafCreator = (childIndex >= beginChild) ? onCreateLeaf : _createMaxSizeLeaf(); - auto child = _createNewSubtree(localBeginIndex, localEndIndex, leafOffset + childOffset, inner->depth() - 1, leafCreator, onBacktrackFromSubtree); - inner->addChild(*child); + auto child = _createNewSubtree(localBeginIndex, localEndIndex, leafOffset + childOffset, root->depth() - 1, leafCreator, onBacktrackFromSubtree); + root->addChild(*child); } // This is only a backtrack, if we actually visited a leaf here. if (endIndex > beginIndex) { - onBacktrackFromSubtree(inner); + onBacktrackFromSubtree(root); } } diff --git a/src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.h b/src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.h index 56e0e8fe..428875c4 100644 --- a/src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.h +++ b/src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.h @@ -6,6 +6,7 @@ #include #include #include +#include "blobstore/implementations/onblocks/datatreestore/LeafHandle.h" namespace blobstore { namespace onblocks { @@ -27,21 +28,25 @@ namespace blobstore { LeafTraverser(datanodestore::DataNodeStore *nodeStore); cpputils::unique_ref traverseAndReturnRoot( - cpputils::unique_ref root, uint32_t beginIndex, uint32_t endIndex, - std::function onExistingLeaf, - std::function onCreateLeaf, - std::function onBacktrackFromSubtree); + cpputils::unique_ref root, uint32_t beginIndex, uint32_t endIndex, + std::function onExistingLeaf, + std::function onCreateLeaf, + std::function onBacktrackFromSubtree); private: datanodestore::DataNodeStore *_nodeStore; cpputils::unique_ref _traverseAndReturnRoot( - cpputils::unique_ref root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, - std::function onExistingLeaf, - std::function onCreateLeaf, - std::function onBacktrackFromSubtree); - void _traverseExistingSubtree(datanodestore::DataNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf, - std::function onExistingLeaf, + cpputils::unique_ref root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, + std::function onExistingLeaf, + std::function onCreateLeaf, + std::function onBacktrackFromSubtree); + void _traverseExistingSubtree(datanodestore::DataInnerNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf, + std::function onExistingLeaf, + std::function onCreateLeaf, + std::function onBacktrackFromSubtree); + void _traverseExistingSubtree(const blockstore::Key &key, uint8_t depth, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool growLastLeaf, + std::function onExistingLeaf, std::function onCreateLeaf, std::function onBacktrackFromSubtree); cpputils::unique_ref _increaseTreeDepth(cpputils::unique_ref root); diff --git a/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h index 758afe52..7f8a7025 100644 --- a/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h +++ b/src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h @@ -4,6 +4,7 @@ #include #include "../datatreestore/DataTree.h" +#include "blobstore/implementations/onblocks/datatreestore/LeafHandle.h" namespace blobstore { namespace onblocks { @@ -21,7 +22,7 @@ public: return _baseTree->maxBytesPerLeaf(); } - void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function onExistingLeaf, std::function onCreateLeaf) { + void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function onExistingLeaf, std::function onCreateLeaf) { return _baseTree->traverseLeaves(beginIndex, endIndex, onExistingLeaf, onCreateLeaf); } diff --git a/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_Performance.cpp b/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_Performance.cpp index 3a5518e4..b2563091 100644 --- a/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_Performance.cpp +++ b/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_Performance.cpp @@ -7,6 +7,7 @@ using blobstore::onblocks::datanodestore::DataLeafNode; using blobstore::onblocks::datanodestore::DataInnerNode; using blobstore::onblocks::datanodestore::DataNode; using blobstore::onblocks::datatreestore::DataTree; +using blobstore::onblocks::datatreestore::LeafHandle; using blockstore::Key; using blockstore::testfake::FakeBlockStore; using cpputils::Data; @@ -15,7 +16,7 @@ using cpputils::make_unique_ref; class DataTreeTest_Performance: public DataTreeTest { public: void Traverse(DataTree *tree, uint64_t beginIndex, uint64_t endIndex) { - tree->traverseLeaves(beginIndex, endIndex, [] (uint32_t /*index*/, DataLeafNode* /*leaf*/) {}, [this] (uint32_t /*index*/) -> Data {return Data(maxChildrenPerInnerNode).FillWithZeroes();}); + tree->traverseLeaves(beginIndex, endIndex, [] (uint32_t /*index*/, LeafHandle /*leaf*/) {}, [this] (uint32_t /*index*/) -> Data {return Data(maxChildrenPerInnerNode).FillWithZeroes();}); } uint64_t maxChildrenPerInnerNode = nodeStore->layout().maxChildrenPerInnerNode(); @@ -83,7 +84,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_All) { Traverse(tree.get(), 0, maxChildrenPerInnerNode); - EXPECT_EQ(maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads all leaves (not the root, because it is already loaded in the tree) + EXPECT_EQ(0u, blockStore->loadedBlocks().size()); // Doesn't actually load the leaves, but returns the keys of the leaves to the callback EXPECT_EQ(0u, blockStore->createdBlocks()); EXPECT_EQ(0u, blockStore->removedBlocks().size()); EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size()); @@ -97,7 +98,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_Some) { Traverse(tree.get(), 3, 5); - EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads both leaves (not the root, because it is already loaded in the tree) + EXPECT_EQ(0u, blockStore->loadedBlocks().size()); // Doesn't actually load the leaves, but returns the keys of the leaves to the callback EXPECT_EQ(0u, blockStore->createdBlocks()); EXPECT_EQ(0u, blockStore->removedBlocks().size()); EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size()); @@ -111,7 +112,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_All) { Traverse(tree.get(), 0, maxChildrenPerInnerNode * maxChildrenPerInnerNode); - EXPECT_EQ(maxChildrenPerInnerNode + maxChildrenPerInnerNode * maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner nodes and all leaves once (not the root, because it is already loaded in the tree) + EXPECT_EQ(maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner nodes. Doesn't load the leaves, but returns the keys of the leaves to the callback EXPECT_EQ(0u, blockStore->createdBlocks()); EXPECT_EQ(0u, blockStore->removedBlocks().size()); EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size()); @@ -125,7 +126,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InOneInner) { Traverse(tree.get(), 3, 5); - EXPECT_EQ(3u, blockStore->loadedBlocks().size()); // Loads inner node and both leaves (not the root, because it is already loaded in the tree) + EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, but returns the keys of the leaves to the callback EXPECT_EQ(0u, blockStore->createdBlocks()); EXPECT_EQ(0u, blockStore->removedBlocks().size()); EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size()); @@ -139,7 +140,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InTwoInner) { Traverse(tree.get(), 3, 3 + maxChildrenPerInnerNode); - EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner node and both leaves (not the root, because it is already loaded in the tree) + EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads both inner node EXPECT_EQ(0u, blockStore->createdBlocks()); EXPECT_EQ(0u, blockStore->removedBlocks().size()); EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size()); @@ -153,7 +154,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_WholeInner) { Traverse(tree.get(), maxChildrenPerInnerNode, 2*maxChildrenPerInnerNode); - EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner node and leaves (not the root, because it is already loaded in the tree)f + EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, but returns the keys of the leaves to the callback EXPECT_EQ(0u, blockStore->createdBlocks()); EXPECT_EQ(0u, blockStore->removedBlocks().size()); EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size()); @@ -209,7 +210,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTree_StartingAtBeginOfChi Traverse(tree.get(), maxChildrenPerInnerNode, 3*maxChildrenPerInnerNode); - EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Inner node and its leaves + EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads inner node and one leaf to check whether we have to grow it. Doesn't load the leaves, but returns the keys of the leaves to the callback. EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // Creates an inner node and its leaves EXPECT_EQ(0u, blockStore->removedBlocks().size()); EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // add children to existing inner node diff --git a/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_ResizeByTraversing.cpp b/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_ResizeByTraversing.cpp index e96775c0..ee5057e1 100644 --- a/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_ResizeByTraversing.cpp +++ b/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_ResizeByTraversing.cpp @@ -19,6 +19,7 @@ using blobstore::onblocks::datanodestore::DataInnerNode; using blobstore::onblocks::datanodestore::DataNode; using blobstore::onblocks::datanodestore::DataNodeLayout; using blobstore::onblocks::datatreestore::DataTree; +using blobstore::onblocks::datatreestore::LeafHandle; using blobstore::onblocks::utils::ceilDivision; using blockstore::Key; using cpputils::Data; @@ -110,7 +111,7 @@ public: void GrowTree(DataTree *tree) { uint64_t maxBytesPerLeaf = tree->maxBytesPerLeaf(); - tree->traverseLeaves(traversalBeginIndex, newNumberOfLeaves, [] (uint32_t, DataLeafNode*){}, [maxBytesPerLeaf] (uint32_t) -> Data { return Data(maxBytesPerLeaf).FillWithZeroes();}); + tree->traverseLeaves(traversalBeginIndex, newNumberOfLeaves, [] (uint32_t, LeafHandle){}, [maxBytesPerLeaf] (uint32_t) -> Data { return Data(maxBytesPerLeaf).FillWithZeroes();}); tree->flush(); } diff --git a/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_TraverseLeaves.cpp b/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_TraverseLeaves.cpp index 28b9bb72..bc669c30 100644 --- a/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_TraverseLeaves.cpp +++ b/test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_TraverseLeaves.cpp @@ -7,6 +7,7 @@ using blobstore::onblocks::datanodestore::DataLeafNode; using blobstore::onblocks::datanodestore::DataInnerNode; using blobstore::onblocks::datanodestore::DataNode; using blobstore::onblocks::datatreestore::DataTree; +using blobstore::onblocks::datatreestore::LeafHandle; using blockstore::Key; using cpputils::unique_ref; @@ -63,8 +64,8 @@ public: void TraverseLeaves(DataNode *root, uint32_t beginIndex, uint32_t endIndex) { root->flush(); auto tree = treeStore.load(root->key()).value(); - tree->traverseLeaves(beginIndex, endIndex, [this] (uint32_t nodeIndex, DataLeafNode *leaf) { - traversor.calledExistingLeaf(leaf, nodeIndex); + tree->traverseLeaves(beginIndex, endIndex, [this] (uint32_t nodeIndex, LeafHandle leaf) { + traversor.calledExistingLeaf(leaf.node(), nodeIndex); }, [this] (uint32_t nodeIndex) -> Data { return traversor.calledCreateLeaf(nodeIndex)->copy(); }); @@ -393,9 +394,9 @@ TEST_F(DataTreeTest_TraverseLeaves, LastLeafIsAlreadyResizedInCallback) { auto root = CreateLeaf(); root->flush(); auto tree = treeStore.load(root->key()).value(); - tree->traverseLeaves(0, 2, [this] (uint32_t leafIndex, DataLeafNode *leaf) { + tree->traverseLeaves(0, 2, [this] (uint32_t leafIndex, LeafHandle leaf) { if (leafIndex == 0) { - EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf->numBytes()); + EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf.node()->numBytes()); } else { EXPECT_TRUE(false) << "only two nodes"; } @@ -408,8 +409,8 @@ TEST_F(DataTreeTest_TraverseLeaves, LastLeafIsAlreadyResizedInCallback_TwoLevel) auto root = CreateFullTwoLevelWithLastLeafSize(5); root->flush(); auto tree = treeStore.load(root->key()).value(); - tree->traverseLeaves(0, nodeStore->layout().maxChildrenPerInnerNode()+1, [this] (uint32_t /*leafIndex*/, DataLeafNode *leaf) { - EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf->numBytes()); + tree->traverseLeaves(0, nodeStore->layout().maxChildrenPerInnerNode()+1, [this] (uint32_t /*leafIndex*/, LeafHandle leaf) { + EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf.node()->numBytes()); }, [this] (uint32_t /*nodeIndex*/) -> Data { return Data(1); });