Implemented BlockStore::blockSizeFromPhysicalBlockSize. This will be used to let the user configure physical block size instead of virtual block size.

src/blockstore/implementations/caching/CachingBlockStore.cpp

@@ -92,5 +92,9 @@ void CachingBlockStore::flush() {
   _cache.flush();
 }
 
+uint64_t CachingBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
+}
+
 }
 }

src/blockstore/implementations/caching/CachingBlockStore.h

@@ -19,6 +19,7 @@ public:
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
   uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
 
   void release(cpputils::unique_ref<Block> block);
 

src/blockstore/implementations/compressing/CompressingBlockStore.h

@@ -20,6 +20,7 @@ public:
     void remove(cpputils::unique_ref<Block> block) override;
     uint64_t numBlocks() const override;
     uint64_t estimateNumFreeBytes() const override;
+    uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
 
 private:
     cpputils::unique_ref<BlockStore> _baseBlockStore;
@@ -77,6 +78,13 @@ uint64_t CompressingBlockStore<Compressor>::estimateNumFreeBytes() const {
     return _baseBlockStore->estimateNumFreeBytes();
 }
 
+template<class Compressor>
+uint64_t CompressingBlockStore<Compressor>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+    //We probably have more since we're compressing, but we don't know exactly how much.
+    //The best we can do is ignore the compression step here.
+    return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
+}
+
 }
 }
 

src/blockstore/implementations/encrypted/EncryptedBlock.h

@@ -31,6 +31,8 @@ public:
   static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey);
   static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key);
 
+  static uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize);
+
   //TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
   EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key, cpputils::Data plaintextWithHeader);
   ~EncryptedBlock();
@@ -198,6 +200,14 @@ cpputils::unique_ref<Block> EncryptedBlock<Cipher>::releaseBlock() {
   return std::move(_baseBlock);
 }
 
+template<class Cipher>
+uint64_t EncryptedBlock<Cipher>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) {
+  if (blockSize <= Cipher::ciphertextSize(HEADER_LENGTH) + sizeof(FORMAT_VERSION_HEADER)) {
+    return 0;
+  }
+  return Cipher::plaintextSize(blockSize - sizeof(FORMAT_VERSION_HEADER)) - HEADER_LENGTH;
+}
+
 }
 }
 

src/blockstore/implementations/encrypted/EncryptedBlockStore.h

@@ -23,6 +23,7 @@ public:
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
   uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
 
   //This function should only be used by test cases
   void __setKey(const typename Cipher::EncryptionKey &encKey);
@@ -90,6 +91,11 @@ void EncryptedBlockStore<Cipher>::__setKey(const typename Cipher::EncryptionKey
   _encKey = encKey;
 }
 
+template<class Cipher>
+uint64_t EncryptedBlockStore<Cipher>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  return EncryptedBlock<Cipher>::blockSizeFromPhysicalBlockSize(_baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize));
+}
+
 }
 }
 

src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp

@@ -57,5 +57,9 @@ uint64_t InMemoryBlockStore::estimateNumFreeBytes() const {
   return cpputils::system::get_total_memory();
 }
 
+uint64_t InMemoryBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  return blockSize;
+}
+
 }
 }

src/blockstore/implementations/inmemory/InMemoryBlockStore.h

@@ -21,6 +21,7 @@ public:
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
   uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
 
 private:
   std::map<std::string, InMemoryBlock> _blocks;

src/blockstore/implementations/ondisk/OnDiskBlock.cpp

@@ -155,5 +155,12 @@ void OnDiskBlock::flush() {
   }
 }
 
+uint64_t OnDiskBlock::blockSizeFromPhysicalBlockSize(uint64_t blockSize) {
+  if(blockSize <= formatVersionHeaderSize()) {
+    return 0;
+  }
+  return blockSize - formatVersionHeaderSize();
+}
+
 }
 }

src/blockstore/implementations/ondisk/OnDiskBlock.h

@@ -22,6 +22,7 @@ public:
   static const std::string FORMAT_VERSION_HEADER_PREFIX;
   static const std::string FORMAT_VERSION_HEADER;
   static unsigned int formatVersionHeaderSize();
+  static uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize);
 
   static boost::optional<cpputils::unique_ref<OnDiskBlock>> LoadFromDisk(const boost::filesystem::path &rootdir, const Key &key);
   static boost::optional<cpputils::unique_ref<OnDiskBlock>> CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data);

src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp

@@ -55,5 +55,9 @@ uint64_t OnDiskBlockStore::estimateNumFreeBytes() const {
   return stat.f_bsize*stat.f_bavail;
 }
 
+uint64_t OnDiskBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  return OnDiskBlock::blockSizeFromPhysicalBlockSize(blockSize);
+}
+
 }
 }

src/blockstore/implementations/ondisk/OnDiskBlockStore.h

@@ -20,6 +20,7 @@ public:
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
   uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
 
 private:
   const boost::filesystem::path _rootdir;

src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.cpp

@@ -60,5 +60,9 @@ uint64_t ParallelAccessBlockStore::estimateNumFreeBytes() const {
   return _baseBlockStore->estimateNumFreeBytes();
 }
 
+uint64_t ParallelAccessBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
+}
+
 }
 }

src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h

@@ -21,6 +21,7 @@ public:
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
   uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
 
 private:
   cpputils::unique_ref<BlockStore> _baseBlockStore;

src/blockstore/implementations/testfake/FakeBlockStore.cpp

@@ -81,5 +81,9 @@ uint64_t FakeBlockStore::estimateNumFreeBytes() const {
   return cpputils::system::get_total_memory();
 }
 
+uint64_t FakeBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
+  return blockSize;
+}
+
 }
 }

src/blockstore/implementations/testfake/FakeBlockStore.h

@@ -36,6 +36,7 @@ public:
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
   uint64_t estimateNumFreeBytes() const override;
+  uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
 
   void updateData(const Key &key, const cpputils::Data &data);
 

src/blockstore/interface/BlockStore.h

@@ -25,6 +25,10 @@ public:
   //TODO Test estimateNumFreeBytes in all block stores
   virtual uint64_t estimateNumFreeBytes() const = 0;
 
+  // Returns, how much space a block has if we allow it to take the given physical block size (i.e. after removing headers, checksums, whatever else).
+  // This can be used to create blocks with a certain physical block size.
+  virtual uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const = 0;
+
   cpputils::unique_ref<Block> create(const cpputils::Data &data) {
     while(true) {
       //TODO Copy (data.copy()) necessary?

test/blockstore/CMakeLists.txt

@@ -7,16 +7,19 @@ set(SOURCES
     interface/BlockTest.cpp
     implementations/testfake/TestFakeBlockStoreTest.cpp
     implementations/inmemory/InMemoryBlockStoreTest.cpp
-    implementations/parallelaccess/ParallelAccessBlockStoreTest.cpp
+    implementations/parallelaccess/ParallelAccessBlockStoreTest_Generic.cpp
+    implementations/parallelaccess/ParallelAccessBlockStoreTest_Specific.cpp
     implementations/compressing/CompressingBlockStoreTest.cpp
     implementations/compressing/compressors/testutils/CompressorTest.cpp
-    implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp
     implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp
-    implementations/ondisk/OnDiskBlockStoreTest.cpp
+    implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp
+    implementations/ondisk/OnDiskBlockStoreTest_Generic.cpp
+    implementations/ondisk/OnDiskBlockStoreTest_Specific.cpp
     implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp
     implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp
     implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp
-    implementations/caching/CachingBlockStoreTest.cpp
+    implementations/caching/CachingBlockStoreTest_Generic.cpp
+    implementations/caching/CachingBlockStoreTest_Specific.cpp
     implementations/caching/cache/QueueMapTest_Values.cpp
     implementations/caching/cache/testutils/MinimalKeyType.cpp
     implementations/caching/cache/testutils/CopyableMovableValueType.cpp

test/blockstore/implementations/caching/CachingBlockStoreTest_Specific.cpp

@@ -0,0 +1,56 @@
+#include <gtest/gtest.h>
+#include "blockstore/implementations/caching/CachingBlockStore.h"
+#include "blockstore/implementations/testfake/FakeBlockStore.h"
+
+using ::testing::Test;
+
+using cpputils::Data;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+
+using blockstore::testfake::FakeBlockStore;
+
+using namespace blockstore::caching;
+
+class CachingBlockStoreTest: public Test {
+public:
+    CachingBlockStoreTest():
+            baseBlockStore(new FakeBlockStore),
+            blockStore(std::move(cpputils::nullcheck(std::unique_ptr<FakeBlockStore>(baseBlockStore)).value()))  {
+    }
+    FakeBlockStore *baseBlockStore;
+    CachingBlockStore blockStore;
+
+    blockstore::Key CreateBlockReturnKey(const Data &initData) {
+        auto block = blockStore.create(initData);
+        block->flush();
+        return block->key();
+    }
+};
+
+TEST_F(CachingBlockStoreTest, PhysicalBlockSize_zerophysical) {
+  EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(0));
+}
+
+TEST_F(CachingBlockStoreTest, PhysicalBlockSize_zerovirtual) {
+    auto key = CreateBlockReturnKey(Data(0));
+    auto base = baseBlockStore->load(key).value();
+    EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(base->size()));
+}
+
+TEST_F(CachingBlockStoreTest, PhysicalBlockSize_negativeboundaries) {
+    // This tests that a potential if/else in blockSizeFromPhysicalBlockSize that catches negative values has the
+    // correct boundary set. We test the highest value that is negative and the smallest value that is positive.
+    auto physicalSizeForVirtualSizeZero = baseBlockStore->load(CreateBlockReturnKey(Data(0))).value()->size();
+    if (physicalSizeForVirtualSizeZero > 0) {
+        EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero - 1));
+    }
+    EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero));
+    EXPECT_EQ(1u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero + 1));
+}
+
+TEST_F(CachingBlockStoreTest, PhysicalBlockSize_positive) {
+    auto key = CreateBlockReturnKey(Data(10*1024));
+    auto base = baseBlockStore->load(key).value();
+    EXPECT_EQ(10*1024u, blockStore.blockSizeFromPhysicalBlockSize(base->size()));
+}

test/blockstore/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp

@@ -30,7 +30,11 @@ public:
   Data data;
 
   blockstore::Key CreateBlockDirectlyWithFixtureAndReturnKey() {
-    return blockStore->create(data)->key();
+    return CreateBlockReturnKey(data);
+  }
+
+  blockstore::Key CreateBlockReturnKey(const Data &initData) {
+    return blockStore->create(initData)->key();
   }
 
   blockstore::Key CreateBlockWriteFixtureToItAndReturnKey() {
@@ -112,3 +116,30 @@ TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentBlockIdFails_WriteSeparately
   auto loaded = blockStore->load(key2);
   EXPECT_EQ(boost::none, loaded);
 }
+
+TEST_F(EncryptedBlockStoreTest, PhysicalBlockSize_zerophysical) {
+  EXPECT_EQ(0u, blockStore->blockSizeFromPhysicalBlockSize(0));
+}
+
+TEST_F(EncryptedBlockStoreTest, PhysicalBlockSize_zerovirtual) {
+  auto key = CreateBlockReturnKey(Data(0));
+  auto base = baseBlockStore->load(key).value();
+  EXPECT_EQ(0u, blockStore->blockSizeFromPhysicalBlockSize(base->size()));
+}
+
+TEST_F(EncryptedBlockStoreTest, PhysicalBlockSize_negativeboundaries) {
+  // This tests that a potential if/else in blockSizeFromPhysicalBlockSize that catches negative values has the
+  // correct boundary set. We test the highest value that is negative and the smallest value that is positive.
+  auto physicalSizeForVirtualSizeZero = baseBlockStore->load(CreateBlockReturnKey(Data(0))).value()->size();
+  if (physicalSizeForVirtualSizeZero > 0) {
+    EXPECT_EQ(0u, blockStore->blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero - 1));
+  }
+  EXPECT_EQ(0u, blockStore->blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero));
+  EXPECT_EQ(1u, blockStore->blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero + 1));
+}
+
+TEST_F(EncryptedBlockStoreTest, PhysicalBlockSize_positive) {
+  auto key = CreateBlockReturnKey(Data(10*1024));
+  auto base = baseBlockStore->load(key).value();
+  EXPECT_EQ(10*1024u, blockStore->blockSizeFromPhysicalBlockSize(base->size()));
+}

test/blockstore/implementations/ondisk/OnDiskBlockStoreTest_Specific.cpp

@@ -0,0 +1,58 @@
+#include <gtest/gtest.h>
+#include "blockstore/implementations/ondisk/OnDiskBlockStore.h"
+#include <cpp-utils/tempfile/TempDir.h>
+
+using ::testing::Test;
+
+using cpputils::TempDir;
+using cpputils::Data;
+using std::ifstream;
+
+using namespace blockstore::ondisk;
+
+class OnDiskBlockStoreTest: public Test {
+public:
+    OnDiskBlockStoreTest():
+    baseDir(),
+    blockStore(baseDir.path()) {
+  }
+  TempDir baseDir;
+  OnDiskBlockStore blockStore;
+
+  blockstore::Key CreateBlockReturnKey(const Data &initData) {
+    return blockStore.create(initData)->key();
+  }
+
+  uint64_t getPhysicalBlockSize(const blockstore::Key &key) {
+    ifstream stream((baseDir.path() / key.ToString()).c_str());
+    stream.seekg(0, stream.end);
+    return stream.tellg();
+  }
+};
+
+TEST_F(OnDiskBlockStoreTest, PhysicalBlockSize_zerophysical) {
+  EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(0));
+}
+
+TEST_F(OnDiskBlockStoreTest, PhysicalBlockSize_zerovirtual) {
+  auto key = CreateBlockReturnKey(Data(0));
+  auto baseSize = getPhysicalBlockSize(key);
+  EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(baseSize));
+}
+
+TEST_F(OnDiskBlockStoreTest, PhysicalBlockSize_negativeboundaries) {
+  // This tests that a potential if/else in blockSizeFromPhysicalBlockSize that catches negative values has the
+  // correct boundary set. We test the highest value that is negative and the smallest value that is positive.
+  auto physicalSizeForVirtualSizeZero = getPhysicalBlockSize(CreateBlockReturnKey(Data(0)));
+  if (physicalSizeForVirtualSizeZero > 0) {
+    EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero - 1));
+  }
+  EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero));
+  EXPECT_EQ(1u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero + 1));
+}
+
+TEST_F(OnDiskBlockStoreTest, PhysicalBlockSize_positive) {
+  auto key = CreateBlockReturnKey(Data(10*1024));
+  auto baseSize = getPhysicalBlockSize(key);
+  EXPECT_EQ(10*1024u, blockStore.blockSizeFromPhysicalBlockSize(baseSize));
+}

test/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreTest_Specific.cpp

@@ -0,0 +1,54 @@
+#include <gtest/gtest.h>
+#include "blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h"
+#include "blockstore/implementations/testfake/FakeBlockStore.h"
+
+using ::testing::Test;
+
+using cpputils::Data;
+using cpputils::unique_ref;
+using cpputils::make_unique_ref;
+
+using blockstore::testfake::FakeBlockStore;
+
+using namespace blockstore::parallelaccess;
+
+class ParallelAccessBlockStoreTest: public Test {
+public:
+    ParallelAccessBlockStoreTest():
+            baseBlockStore(new FakeBlockStore),
+            blockStore(std::move(cpputils::nullcheck(std::unique_ptr<FakeBlockStore>(baseBlockStore)).value()))  {
+    }
+    FakeBlockStore *baseBlockStore;
+    ParallelAccessBlockStore blockStore;
+
+    blockstore::Key CreateBlockReturnKey(const Data &initData) {
+        return blockStore.create(initData)->key();
+    }
+};
+
+TEST_F(ParallelAccessBlockStoreTest, PhysicalBlockSize_zerophysical) {
+    EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(0));
+}
+
+TEST_F(ParallelAccessBlockStoreTest, PhysicalBlockSize_zerovirtual) {
+    auto key = CreateBlockReturnKey(Data(0));
+    auto base = baseBlockStore->load(key).value();
+    EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(base->size()));
+}
+
+TEST_F(ParallelAccessBlockStoreTest, PhysicalBlockSize_negativeboundaries) {
+    // This tests that a potential if/else in blockSizeFromPhysicalBlockSize that catches negative values has the
+    // correct boundary set. We test the highest value that is negative and the smallest value that is positive.
+    auto physicalSizeForVirtualSizeZero = baseBlockStore->load(CreateBlockReturnKey(Data(0))).value()->size();
+    if (physicalSizeForVirtualSizeZero > 0) {
+        EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero - 1));
+    }
+    EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero));
+    EXPECT_EQ(1u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero + 1));
+}
+
+TEST_F(ParallelAccessBlockStoreTest, PhysicalBlockSize_positive) {
+    auto key = CreateBlockReturnKey(Data(10*1024));
+    auto base = baseBlockStore->load(key).value();
+    EXPECT_EQ(10*1024u, blockStore.blockSizeFromPhysicalBlockSize(base->size()));
+}

test/blockstore/interface/helpers/BlockStoreWithRandomKeysTest.cpp

@@ -31,6 +31,7 @@ public:
   void remove(unique_ref<Block> block) {UNUSED(block);}
   MOCK_CONST_METHOD0(numBlocks, uint64_t());
   MOCK_CONST_METHOD0(estimateNumFreeBytes, uint64_t());
+  MOCK_CONST_METHOD1(blockSizeFromPhysicalBlockSize, uint64_t(uint64_t));
 };
 
 class BlockMock: public Block {