More preparations for not loading blocks in write() calls

src/blobstore/implementations/onblocks/BlobOnBlocks.cpp

@@ -41,7 +41,7 @@ void BlobOnBlocks::resize(uint64_t numBytes) {
   _sizeCache = numBytes;
 }
 
-void BlobOnBlocks::_traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function<void (uint64_t leafOffset, DataLeafNode *leaf, uint32_t begin, uint32_t count)> onExistingLeaf, function<Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const {
+void BlobOnBlocks::_traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function<void (uint64_t leafOffset, LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, function<Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const {
   unique_lock<mutex> lock(_mutex); // TODO Multiple traverse calls in parallel?
   uint64_t endByte = beginByte + sizeBytes;
   uint64_t maxBytesPerLeaf = _datatree->maxBytesPerLeaf();
@@ -49,18 +49,19 @@ void BlobOnBlocks::_traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, funct
   uint32_t endLeaf = utils::ceilDivision(endByte, maxBytesPerLeaf);
   bool blobIsGrowingFromThisTraversal = false;
   auto _onExistingLeaf = [&onExistingLeaf, beginByte, endByte, endLeaf, maxBytesPerLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex, LeafHandle leafHandle) {
-      auto leaf = leafHandle.node();
       uint64_t indexOfFirstLeafByte = leafIndex * maxBytesPerLeaf;
       ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
-      ASSERT(leafIndex == endLeaf-1 || leaf->numBytes() == maxBytesPerLeaf, "All leafes but the rightmost one have to have maximal size");
       uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
       uint32_t dataEnd = std::min(maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
-      if (leafIndex == endLeaf-1 && leaf->numBytes() < dataEnd) {
+      if (leafIndex == endLeaf-1) {
-        // If we are traversing an area that didn't exist before (i.e. in the area of the last leaf that wasn't used before), then the last leaf might have a wrong size. We have to fix it.
+        auto leaf = leafHandle.node();
-        leaf->resize(dataEnd);
+        if (leaf->numBytes() < dataEnd) {
-        blobIsGrowingFromThisTraversal = true;
+          // If we are traversing an area that didn't exist before (i.e. in the area of the last leaf that wasn't used before), then the last leaf might have a wrong size. We have to fix it.
+          leaf->resize(dataEnd);
+          blobIsGrowingFromThisTraversal = true;
+        }
       }
-      onExistingLeaf(indexOfFirstLeafByte, leaf, dataBegin, dataEnd-dataBegin);
+      onExistingLeaf(indexOfFirstLeafByte, std::move(leafHandle), dataBegin, dataEnd-dataBegin);
   };
   auto _onCreateLeaf = [&onCreateLeaf, maxBytesPerLeaf, beginByte, firstLeaf, endByte, endLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex) -> Data {
       blobIsGrowingFromThisTraversal = true;
@@ -111,10 +112,10 @@ uint64_t BlobOnBlocks::tryRead(void *target, uint64_t offset, uint64_t count) co
 }
 
 void BlobOnBlocks::_read(void *target, uint64_t offset, uint64_t count) const {
-  auto onExistingLeaf = [target, offset, count] (uint64_t indexOfFirstLeafByte, const DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
+  auto onExistingLeaf = [target, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
       ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Writing to target out of bounds");
       //TODO Simplify formula, make it easier to understand
-      leaf->read((uint8_t*)target + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
+      leaf.node()->read((uint8_t*)target + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
   };
   auto onCreateLeaf = [] (uint64_t /*beginByte*/, uint32_t /*count*/) -> Data {
       ASSERT(false, "Reading shouldn't create new leaves.");
@@ -123,10 +124,10 @@ void BlobOnBlocks::_read(void *target, uint64_t offset, uint64_t count) const {
 }
 
 void BlobOnBlocks::write(const void *source, uint64_t offset, uint64_t count) {
-  auto onExistingLeaf = [source, offset, count] (uint64_t indexOfFirstLeafByte, DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
+  auto onExistingLeaf = [source, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
       ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Reading from source out of bounds");
       //TODO Simplify formula, make it easier to understand
-      leaf->write((uint8_t*)source + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
+      leaf.node()->write((uint8_t*)source + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
   };
   auto onCreateLeaf = [source, offset, count] (uint64_t beginByte, uint32_t numBytes) -> Data {
       ASSERT(beginByte >= offset && beginByte-offset <= count && beginByte-offset+numBytes <= count, "Reading from source out of bounds");

src/blobstore/implementations/onblocks/BlobOnBlocks.h

@@ -3,6 +3,7 @@
 #define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_BLOBONBLOCKS_H_
 
 #include "../../interface/Blob.h"
+#include "datatreestore/LeafHandle.h"
 
 #include <memory>
 #include <boost/optional.hpp>
@@ -38,7 +39,7 @@ public:
 private:
 
   void _read(void *target, uint64_t offset, uint64_t count) const;
-  void _traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function<void (uint64_t leafOffset, datanodestore::DataLeafNode *leaf, uint32_t begin, uint32_t count)> onExistingLeaf, std::function<cpputils::Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const;
+  void _traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function<void (uint64_t leafOffset, datatreestore::LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, std::function<cpputils::Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const;
 
   cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> _datatree;
   mutable boost::optional<uint64_t> _sizeCache;

src/blobstore/implementations/onblocks/datatreestore/LeafHandle.cpp

@@ -1 +1,40 @@
 #include "LeafHandle.h"
+#include "../datanodestore/DataLeafNode.h"
+#include "../datanodestore/DataNodeStore.h"
+
+using cpputils::WithOwnership;
+using cpputils::WithoutOwnership;
+using boost::none;
+using cpputils::dynamic_pointer_move;
+using blobstore::onblocks::datanodestore::DataLeafNode;
+using blobstore::onblocks::datanodestore::DataNodeStore;
+using blockstore::Key;
+
+namespace blobstore {
+    namespace onblocks {
+        namespace datatreestore {
+
+            LeafHandle::LeafHandle(DataNodeStore *nodeStore, const Key &key)
+                    : _nodeStore(nodeStore), _key(key), _leaf(cpputils::null<DataLeafNode>()) {
+            }
+
+            LeafHandle::LeafHandle(DataNodeStore *nodeStore, DataLeafNode *node)
+                    : _nodeStore(nodeStore), _key(node->key()),
+                      _leaf(WithoutOwnership<DataLeafNode>(node)) {
+            }
+
+            DataLeafNode *LeafHandle::node() {
+                if (_leaf.get() == nullptr) {
+                    auto loaded = _nodeStore->load(_key);
+                    ASSERT(loaded != none, "Leaf not found");
+                    auto leaf = dynamic_pointer_move<DataLeafNode>(*loaded);
+                    ASSERT(leaf != none, "Loaded leaf is not leaf node");
+
+                    _leaf = WithOwnership(std::move(*leaf));
+                }
+
+                return _leaf.get();
+            }
+        }
+    }
+}

src/blobstore/implementations/onblocks/datatreestore/LeafHandle.h

@@ -4,41 +4,27 @@
 
 #include <cpp-utils/macros.h>
 #include <cpp-utils/pointer/optional_ownership_ptr.h>
-#include "blobstore/implementations/onblocks/datanodestore/DataNodeStore.h"
+#include <blockstore/utils/Key.h>
-#include "blobstore/implementations/onblocks/datanodestore/DataLeafNode.h"
 
 namespace blobstore {
     namespace onblocks {
+        namespace datanodestore {
+            class DataNodeStore;
+            class DataLeafNode;
+        }
         namespace datatreestore {
 
             class LeafHandle final {
             public:
-                LeafHandle(datanodestore::DataNodeStore *nodeStore, const blockstore::Key &key)
+                LeafHandle(datanodestore::DataNodeStore *nodeStore, const blockstore::Key &key);
-                    :_nodeStore(nodeStore), _key(key), _leaf(cpputils::null<datanodestore::DataLeafNode>()) {
+                LeafHandle(datanodestore::DataNodeStore *nodeStore, datanodestore::DataLeafNode *node);
-                }
-
-                LeafHandle(datanodestore::DataNodeStore *nodeStore, datanodestore::DataLeafNode *node)
-                        : _nodeStore(nodeStore), _key(node->key()), _leaf(cpputils::WithoutOwnership<datanodestore::DataLeafNode>(node)) {
-                }
-
                 LeafHandle(LeafHandle &&rhs) = default;
 
                 const blockstore::Key &key() {
                     return _key;
                 }
 
-                datanodestore::DataLeafNode *node() {
+                datanodestore::DataLeafNode *node();
-                    if (_leaf.get() == nullptr) {
-                        auto loaded = _nodeStore->load(_key);
-                        ASSERT(loaded != boost::none, "Leaf not found");
-                        auto leaf = cpputils::dynamic_pointer_move<datanodestore::DataLeafNode>(*loaded);
-                        ASSERT(leaf != boost::none, "Loaded leaf is not leaf node");
-
-                        _leaf = cpputils::WithOwnership(std::move(*leaf));
-                    }
-
-                    return _leaf.get();
-                }
 
             private:
                 datanodestore::DataNodeStore *_nodeStore;