Redesign DataNodes

src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h

@@ -10,6 +10,7 @@ namespace onblocks {
 
 class BlobStoreOnBlocks: public BlobStore {
 public:
+  //Should be a multiple of 16. The DataNodeView classes have a header of 16 bytes and the block key length (inner data nodes store a list of block keys) is 16 bytes.
   static constexpr size_t BLOCKSIZE = 4096;
 
   BlobStoreOnBlocks(std::unique_ptr<blockstore::BlockStore> blockStore);

src/blobstore/implementations/onblocks/impl/DataInnerNode.cpp

@@ -2,20 +2,93 @@
 
 using std::unique_ptr;
 using blockstore::Block;
+using blockstore::Data;
 
 namespace blobstore {
 namespace onblocks {
 
-DataInnerNode::DataInnerNode(unique_ptr<Block> block)
+DataInnerNode::DataInnerNode(DataNodeView view)
-: DataNode(std::move(block)) {
+: DataNode(std::move(view)) {
 }
 
 DataInnerNode::~DataInnerNode() {
 }
 
-void DataInnerNode::InitializeEmptyInnerNode() {
+void DataInnerNode::InitializeNewInnerNode() {
-  InnerNodeHeader* header = (InnerNodeHeader*)_block->data();
+  *_node.MagicNumber() = _node.magicNumberNodeWithChildren;
-  header->nodeHeader.magicNumber = DataNode::magicNumberInnerNode;
+  *_node.Size() = 0;
+}
+
+void DataInnerNode::read(off_t offset, size_t count, Data *result) {
+  assert(count <= result->size());
+  const uint64_t end = offset + count;
+  assert(end <= numBytesInThisNode());
+
+  uint8_t *target = (uint8_t*)result->data();
+
+  ChildEntry *child = ChildContainingFirstByteAfterOffset(offset);
+  off_t blockrelative_offset = offset - numBytesInLeftwardSiblings(child);
+  uint64_t already_read_bytes = readFromChild(child, blockrelative_offset, count, target);
+  while(numBytesInChildAndLeftwardSiblings(child) < end) {
+    ++child;
+    already_read_bytes += readFromChild(child, 0, count, target + already_read_bytes);
+  };
+  assert(already_read_bytes == count);
+}
+
+uint64_t DataInnerNode::readFromChild(const ChildEntry *child, off_t inner_offset, size_t count, uint8_t *target) {
+  uint64_t readable_bytes = std::min(count, numBytesInChild(child) - inner_offset);
+
+  //TODO READ...
+
+  return readable_bytes;
+}
+
+DataInnerNode::ChildEntry *DataInnerNode::ChildContainingFirstByteAfterOffset(off_t offset) {
+  uint32_t offset_blocks = offset / _node.BLOCKSIZE_BYTES;
+
+  return
+    std::upper_bound(ChildrenBegin(), ChildrenEnd(), offset_blocks, [](uint32_t offset_blocks, const ChildEntry &child) {
+      return offset_blocks < child.numBlocksInThisAndLeftwardNodes;
+    });
+}
+
+uint64_t DataInnerNode::numBytesInThisNode() {
+  return numBytesInChildAndLeftwardSiblings(ChildrenLast());
+}
+
+uint64_t DataInnerNode::numBytesInChild(const ChildEntry *child) {
+  return numBytesInChildAndLeftwardSiblings(child) - numBytesInLeftwardSiblings(child);
+}
+
+uint64_t DataInnerNode::numBytesInLeftwardSiblings(const ChildEntry *child) {
+  if (child == ChildrenBegin()) {
+    return 0;
+  }
+  return numBytesInChildAndLeftwardSiblings(child-1);
+}
+
+uint64_t DataInnerNode::numBytesInChildAndLeftwardSiblings(const ChildEntry *child) {
+  return (uint64_t)child->numBlocksInThisAndLeftwardNodes * _node.BLOCKSIZE_BYTES;
+}
+
+DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() {
+  return _node.DataBegin<ChildEntry>();
+}
+
+DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() {
+  return ChildrenBegin() + *_node.Size();
+}
+
+DataInnerNode::ChildEntry *DataInnerNode::ChildrenLast() {
+  return ChildrenEnd()-1;
+}
+
+void DataInnerNode::write(off_t offset, size_t count, const Data &data) {
+  //assert(count <= data.size());
+  //assert(offset+count <= _node->DATASIZE_BYTES);
+  //std::memcpy(_node->DataBegin()+offset, result.data(), count);
+
 }
 
 }

src/blobstore/implementations/onblocks/impl/DataInnerNode.h

@@ -9,14 +9,32 @@ namespace onblocks {
 
 class DataInnerNode: public DataNode {
 public:
-  DataInnerNode(std::unique_ptr<blockstore::Block> block);
+  DataInnerNode(DataNodeView block);
   virtual ~DataInnerNode();
 
-  struct InnerNodeHeader {
+  struct ChildEntry {
-    NodeHeader nodeHeader;
+    uint32_t numBlocksInThisAndLeftwardNodes;
+    uint8_t key[Key::KEYLENGTH_BINARY];
   };
 
-  void InitializeEmptyInnerNode();
+  void InitializeNewInnerNode();
+
+  void read(off_t offset, size_t count, blockstore::Data *result) override;
+  void write(off_t offset, size_t count, const blockstore::Data &data) override;
+
+  uint64_t numBytesInThisNode() override;
+
+private:
+  ChildEntry *ChildrenBegin();
+  ChildEntry *ChildrenEnd();
+  ChildEntry *ChildrenLast();
+
+  uint64_t readFromChild(const ChildEntry *child, off_t inner_offset, size_t count, uint8_t *target);
+
+  ChildEntry *ChildContainingFirstByteAfterOffset(off_t offset);
+  uint64_t numBytesInChildAndLeftwardSiblings(const ChildEntry *child);
+  uint64_t numBytesInLeftwardSiblings(const ChildEntry *child);
+  uint64_t numBytesInChild(const ChildEntry *child);
 };
 
 }

src/blobstore/implementations/onblocks/impl/DataLeafNode.cpp

@@ -2,20 +2,37 @@
 
 using std::unique_ptr;
 using blockstore::Block;
+using blockstore::Data;
 
 namespace blobstore {
 namespace onblocks {
 
-DataLeafNode::DataLeafNode(unique_ptr<Block> block)
+DataLeafNode::DataLeafNode(DataNodeView view)
-: DataNode(std::move(block)) {
+: DataNode(std::move(view)) {
 }
 
 DataLeafNode::~DataLeafNode() {
 }
 
-void DataLeafNode::InitializeEmptyLeaf() {
+void DataLeafNode::read(off_t offset, size_t count, Data *result) {
-  LeafHeader *header = (LeafHeader*)_block->data();
+  assert(count <= result->size());
-  header->nodeHeader.magicNumber = DataNode::magicNumberLeaf;
+  assert(offset+count <= _node.DATASIZE_BYTES);
+  std::memcpy(result->data(), _node.DataBegin<unsigned char>()+offset, count);
+}
+
+void DataLeafNode::write(off_t offset, size_t count, const Data &data) {
+  assert(count <= data.size());
+  assert(offset+count <= _node.DATASIZE_BYTES);
+  std::memcpy(_node.DataBegin<unsigned char>()+offset, data.data(), count);
+}
+
+void DataLeafNode::InitializeNewLeafNode() {
+  *_node.MagicNumber() = _node.magicNumberLeaf;
+  *_node.Size() = 0;
+}
+
+uint64_t DataLeafNode::numBytesInThisNode() {
+  return *_node.Size();
 }
 
 }

src/blobstore/implementations/onblocks/impl/DataLeafNode.h

@@ -9,14 +9,16 @@ namespace onblocks {
 
 class DataLeafNode: public DataNode {
 public:
-  DataLeafNode(std::unique_ptr<blockstore::Block> block);
+  DataLeafNode(DataNodeView block);
   virtual ~DataLeafNode();
 
-  struct LeafHeader {
+  void read(off_t offset, size_t count, blockstore::Data *result) override;
-    NodeHeader nodeHeader;
+  void write(off_t offset, size_t count, const blockstore::Data &data) override;
-  };
+
+  void InitializeNewLeafNode();
+
+  uint64_t numBytesInThisNode() override;
 
-  void InitializeEmptyLeaf();
 };
 
 }

src/blobstore/implementations/onblocks/impl/DataNode.cpp

@@ -12,40 +12,38 @@ using std::runtime_error;
 namespace blobstore {
 namespace onblocks {
 
-DataNode::DataNode(unique_ptr<Block> block)
+DataNode::DataNode(DataNodeView node)
-: _block(std::move(block)) {
+: _node(std::move(node)) {
 }
 
 DataNode::~DataNode() {
 }
 
-void DataNode::flush() {
-  _block->flush();
-}
-
 unique_ptr<DataNode> DataNode::load(unique_ptr<Block> block) {
-  NodeHeader *header = (NodeHeader*)block->data();
+  DataNodeView node(std::move(block));
-  if (header->magicNumber == magicNumberInnerNode) {
+
-    return make_unique<DataInnerNode>(std::move(block));
+  if (*node.MagicNumber() == node.magicNumberNodeWithChildren) {
-  } else if (header->magicNumber == magicNumberLeaf) {
+    return make_unique<DataInnerNode>(std::move(node));
-    return make_unique<DataLeafNode>(std::move(block));
+  } else if (*node.MagicNumber() == node.magicNumberLeaf) {
+    return make_unique<DataLeafNode>(std::move(node));
   } else {
     //TODO Better exception
     throw runtime_error("Invalid node magic number");
   }
 }
 
-unique_ptr<DataInnerNode> DataNode::initializeNewInnerNode(unique_ptr<Block> block) {
+/*
+unique_ptr<DataInnerNode> DataNodeView::initializeNewInnerNode(unique_ptr<Block> block) {
   auto newNode = make_unique<DataInnerNode>(std::move(block));
   newNode->InitializeEmptyInnerNode();
   return newNode;
 }
 
-unique_ptr<DataLeafNode> DataNode::initializeNewLeafNode(unique_ptr<Block> block) {
+unique_ptr<DataLeafNode> DataNodeView::initializeNewLeafNode(unique_ptr<Block> block) {
   auto newNode = make_unique<DataLeafNode>(std::move(block));
   newNode->InitializeEmptyLeaf();
   return newNode;
-}
+}*/
 
 }
 }

src/blobstore/implementations/onblocks/impl/DataNode.h

@@ -2,40 +2,33 @@
 #ifndef BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_DATANODE_H_
 #define BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_DATANODE_H_
 
-#include "blockstore/interface/Block.h"
+#include "DataNodeView.h"
 
-#include <memory>
+#include "blockstore/utils/Data.h"
 
 namespace blobstore {
 namespace onblocks {
-class DataInnerNode;
-class DataLeafNode;
 
 class DataNode {
 public:
   virtual ~DataNode();
 
-  static constexpr unsigned char magicNumberInnerNode = 0x01;
+  //TODO MAke read, numBytesInThisNode const
-  static constexpr unsigned char magicNumberLeaf = 0x02;
+  virtual void read(off_t offset, size_t count, blockstore::Data *result) = 0;
-  struct NodeHeader {
+  virtual void write(off_t offset, size_t count, const blockstore::Data &data) = 0;
-    unsigned char magicNumber;
-  };
 
-  void flush();
+  virtual uint64_t numBytesInThisNode() = 0;
 
   static std::unique_ptr<DataNode> load(std::unique_ptr<blockstore::Block> block);
 
-  static std::unique_ptr<DataInnerNode> initializeNewInnerNode(std::unique_ptr<blockstore::Block> block);
-  static std::unique_ptr<DataLeafNode> initializeNewLeafNode(std::unique_ptr<blockstore::Block> block);
-
 protected:
-  DataNode(std::unique_ptr<blockstore::Block> block);
+  DataNode(DataNodeView block);
-
-  std::unique_ptr<blockstore::Block> _block;
 
+  DataNodeView _node;
 };
 
 }
 }
 
+
 #endif

src/blobstore/implementations/onblocks/impl/DataNodeView.h

@@ -0,0 +1,93 @@
+#pragma once
+#ifndef BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_DATANODEVIEW_H_
+#define BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_DATANODEVIEW_H_
+
+#include "blockstore/interface/Block.h"
+#include "blobstore/implementations/onblocks/BlobStoreOnBlocks.h"
+
+#include "fspp/utils/macros.h"
+
+#include <memory>
+
+namespace blobstore {
+namespace onblocks {
+
+class DataNodeView {
+public:
+  DataNodeView(std::unique_ptr<blockstore::Block> block): _block(std::move(block)) {}
+  virtual ~DataNodeView() {}
+
+  DataNodeView(DataNodeView &&rhs) = default;
+
+  //Total size of the header
+  static constexpr unsigned int HEADERSIZE_BYTES = 8;
+  //Where in the header is the magic number
+  static constexpr unsigned int MAGICNUMBER_OFFSET_BYTES = 0;
+  //Where in the header is the size field (for inner nodes: number of children, for leafs: content data size)
+  static constexpr unsigned int SIZE_OFFSET_BYTES = 4;
+
+  //How big is one blob in total (header + data)
+  static constexpr unsigned int BLOCKSIZE_BYTES = BlobStoreOnBlocks::BLOCKSIZE;
+  //How much space is there for data
+  static constexpr unsigned int DATASIZE_BYTES = BLOCKSIZE_BYTES - HEADERSIZE_BYTES;
+
+  static constexpr unsigned char magicNumberNodeWithChildren = 0x01;
+  static constexpr unsigned char magicNumberLeaf = 0x02;
+  static constexpr unsigned char magicNumberRootWithChildren = 0x03;
+  static constexpr unsigned char magicNumberRootLeaf = 0x04;
+
+  const uint8_t *MagicNumber() const {
+    return GetOffset<MAGICNUMBER_OFFSET_BYTES, uint8_t>();
+  }
+
+  uint8_t *MagicNumber() {
+    return const_cast<uint8_t*>(const_cast<const DataNodeView*>(this)->MagicNumber());
+  }
+
+  const uint32_t *Size() const {
+    return GetOffset<SIZE_OFFSET_BYTES, uint32_t>();
+  }
+
+  uint32_t *Size() {
+    return const_cast<uint32_t*>(const_cast<const DataNodeView*>(this)->Size());
+  }
+
+  template<typename Entry>
+  const Entry *DataBegin() const {
+    return GetOffset<HEADERSIZE_BYTES, Entry>();
+  }
+
+  template<typename Entry>
+  Entry *DataBegin() {
+    return const_cast<Entry*>(const_cast<const DataNodeView*>(this)->DataBegin<Entry>());
+  }
+
+  template<typename Entry>
+  const Entry *DataEnd() const {
+    constexpr unsigned int NUM_ENTRIES = DATASIZE_BYTES / sizeof(Entry);
+    return DataBegin<Entry>() + NUM_ENTRIES;
+  }
+
+  template<typename Entry>
+  Entry *DataEnd() {
+    return const_cast<Entry*>(const_cast<const DataNodeView*>(this)->DataEnd<Entry>());
+  }
+
+protected:
+
+  template<int offset, class Type>
+  const Type *GetOffset() const {
+    return (Type*)(((const int8_t*)_block->data())+offset);
+  }
+
+private:
+  DISALLOW_COPY_AND_ASSIGN(DataNodeView);
+
+  std::unique_ptr<blockstore::Block> _block;
+
+};
+
+}
+}
+
+#endif

src/test/blobstore/implementations/onblocks/impl/DataNodeTest.cpp

@@ -24,27 +24,12 @@ public:
 
 #define EXPECT_IS_PTR_TYPE(Type, ptr) EXPECT_NE(nullptr, dynamic_cast<Type*>(ptr)) << "Given pointer cannot be cast to the given type"
 
-TEST_F(DataNodeTest, InitializeNewLeafNodeCreatesLeafNodeObject) {
-  auto block = blockStore->create(BlobStoreOnBlocks::BLOCKSIZE);
-  Key key = block.key;
-  auto leafNode = DataNode::initializeNewLeafNode(std::move(block.block));
-
-  EXPECT_IS_PTR_TYPE(DataLeafNode, leafNode.get());
-}
-
-TEST_F(DataNodeTest, InitializeNewInnerNodeCreatesInnerNodeObject) {
-  auto block = blockStore->create(BlobStoreOnBlocks::BLOCKSIZE);
-  Key key = block.key;
-  auto innerNode = DataNode::initializeNewInnerNode(std::move(block.block));
-
-  EXPECT_IS_PTR_TYPE(DataInnerNode, innerNode.get());
-}
-
 TEST_F(DataNodeTest, LeafNodeIsRecognizedAfterStoreAndLoad) {
   auto block = blockStore->create(BlobStoreOnBlocks::BLOCKSIZE);
   Key key = block.key;
-  auto node = DataNode::initializeNewLeafNode(std::move(block.block));
+  {
-  node->flush();
+    DataLeafNode(std::move(block.block)).InitializeNewLeafNode();
+  }
 
   auto loaded_node = DataNode::load(blockStore->load(key));
 
@@ -54,8 +39,9 @@ TEST_F(DataNodeTest, LeafNodeIsRecognizedAfterStoreAndLoad) {
 TEST_F(DataNodeTest, InnerNodeIsRecognizedAfterStoreAndLoad) {
   auto block = blockStore->create(BlobStoreOnBlocks::BLOCKSIZE);
   Key key = block.key;
-  auto node = DataNode::initializeNewInnerNode(std::move(block.block));
+  {
-  node->flush();
+    DataInnerNode(std::move(block.block)).InitializeNewInnerNode();
+  }
 
   auto loaded_node = DataNode::load(blockStore->load(key));
 
@@ -65,10 +51,13 @@ TEST_F(DataNodeTest, InnerNodeIsRecognizedAfterStoreAndLoad) {
 TEST_F(DataNodeTest, DataNodeCrashesOnLoadIfMagicNumberIsWrong) {
   auto block = blockStore->create(BlobStoreOnBlocks::BLOCKSIZE);
   Key key = block.key;
-  DataNode::NodeHeader* header = (DataNode::NodeHeader*)block.block->data();
+  {
-  header->magicNumber = 0xFF; // this is an invalid magic number
+    DataNodeView view(std::move(block.block));
+    *view.MagicNumber() = 0xFF; // this is an invalid magic number
+  }
 
+  auto loaded_block = blockStore->load(key);
   EXPECT_ANY_THROW(
-    DataNode::load(std::move(block.block))
+    DataNode::load(std::move(loaded_block))
   );
 }