- Fix CachingBlockStore2_Specific test cases to actually test CachingBlockStore2

- Remove old CachingBlockStore
This commit is contained in:
Sebastian Messmer 2017-08-22 00:03:04 +01:00
parent 30cec27bd8
commit ffc0b5195c
15 changed files with 27 additions and 503 deletions

View File

@ -17,14 +17,11 @@ set(SOURCES
implementations/compressing/compressors/Gzip.cpp
implementations/encrypted/EncryptedBlockStore2.cpp
implementations/ondisk/OnDiskBlockStore2.cpp
implementations/caching/CachingBlockStore.cpp
implementations/caching2/CachingBlockStore2.cpp
implementations/caching/CachingBlockStore2.cpp
implementations/caching/cache/PeriodicTask.cpp
implementations/caching/cache/CacheEntry.cpp
implementations/caching/cache/Cache.cpp
implementations/caching/cache/QueueMap.cpp
implementations/caching/CachedBlock.cpp
implementations/caching/NewBlock.cpp
implementations/low2highlevel/LowToHighLevelBlock.cpp
implementations/low2highlevel/LowToHighLevelBlockStore.cpp
implementations/versioncounting/VersionCountingBlockStore2.cpp

View File

@ -1,46 +0,0 @@
#include "CachedBlock.h"
#include "CachingBlockStore.h"
using cpputils::unique_ref;
namespace blockstore {
namespace caching {
CachedBlock::CachedBlock(unique_ref<Block> baseBlock, CachingBlockStore *blockStore)
:Block(baseBlock->key()),
_blockStore(blockStore),
_baseBlock(std::move(baseBlock)) {
}
CachedBlock::~CachedBlock() {
if (_baseBlock.is_valid()) {
_blockStore->release(std::move(_baseBlock));
}
}
const void *CachedBlock::data() const {
return _baseBlock->data();
}
void CachedBlock::write(const void *source, uint64_t offset, uint64_t size) {
return _baseBlock->write(source, offset, size);
}
void CachedBlock::flush() {
return _baseBlock->flush();
}
size_t CachedBlock::size() const {
return _baseBlock->size();
}
void CachedBlock::resize(size_t newSize) {
return _baseBlock->resize(newSize);
}
unique_ref<Block> CachedBlock::releaseBlock() {
return std::move(_baseBlock);
}
}
}

View File

@ -1,39 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/pointer/unique_ref.h>
namespace blockstore {
namespace caching {
class CachingBlockStore;
class CachedBlock final: public Block {
public:
//TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
CachedBlock(cpputils::unique_ref<Block> baseBlock, CachingBlockStore *blockStore);
~CachedBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref<Block> releaseBlock();
private:
CachingBlockStore *_blockStore;
cpputils::unique_ref<Block> _baseBlock;
DISALLOW_COPY_AND_ASSIGN(CachedBlock);
};
}
}
#endif

View File

@ -1,134 +0,0 @@
#include "CachedBlock.h"
#include "NewBlock.h"
#include "CachingBlockStore.h"
#include "../../interface/Block.h"
#include <algorithm>
#include <cpp-utils/pointer/cast.h>
#include <cpp-utils/assert/assert.h>
using cpputils::dynamic_pointer_move;
using cpputils::Data;
using boost::optional;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::none;
using std::mutex;
using std::unique_lock;
namespace blockstore {
namespace caching {
CachingBlockStore::CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore)
:_baseBlockStore(std::move(baseBlockStore)), _newBlocks(), _cache() {
}
Key CachingBlockStore::createKey() {
return _baseBlockStore->createKey();
}
optional<unique_ref<Block>> CachingBlockStore::tryCreate(const Key &key, Data data) {
ASSERT(_cache.pop(key) == none, "Key already exists in cache");
//TODO Shouldn't we return boost::none if the key already exists?
//TODO Key can also already exist but not be in the cache right now.
return unique_ref<Block>(make_unique_ref<CachedBlock>(make_unique_ref<NewBlock>(key, std::move(data), this), this));
}
optional<unique_ref<Block>> CachingBlockStore::load(const Key &key) {
optional<unique_ref<Block>> optBlock = _cache.pop(key);
//TODO an optional<> class with .getOrElse() would make this code simpler. boost::optional<>::value_or_eval didn't seem to work with unique_ptr members.
if (optBlock != none) {
return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*optBlock), this));
} else {
auto block = _baseBlockStore->load(key);
if (block == none) {
return none;
} else {
return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*block), this));
}
}
}
unique_ref<Block> CachingBlockStore::overwrite(const Key &key, cpputils::Data data) {
optional<unique_ref<Block>> optBlock = _cache.pop(key);
if (optBlock != none) {
if ((*optBlock)->size() != data.size()) {
(*optBlock)->resize(data.size());
}
(*optBlock)->write(data.data(), 0, data.size());
return make_unique_ref<CachedBlock>(std::move(*optBlock), this);
} else {
auto block = _baseBlockStore->overwrite(key, std::move(data));
return make_unique_ref<CachedBlock>(std::move(block), this);
}
}
void CachingBlockStore::remove(cpputils::unique_ref<Block> block) {
auto cached_block = dynamic_pointer_move<CachedBlock>(block);
ASSERT(cached_block != none, "Passed block is not a CachedBlock");
auto baseBlock = (*cached_block)->releaseBlock();
auto baseNewBlock = dynamic_pointer_move<NewBlock>(baseBlock);
if (baseNewBlock != none) {
(*baseNewBlock)->remove();
} else {
_baseBlockStore->remove(std::move(baseBlock));
}
}
void CachingBlockStore::remove(const Key &key) {
auto fromCache = _cache.pop(key);
if (fromCache != none) {
remove(make_unique_ref<CachedBlock>(std::move(*fromCache), this));
} else {
_baseBlockStore->remove(key);
}
}
uint64_t CachingBlockStore::numBlocks() const {
return _baseBlockStore->numBlocks() + _newBlocks.size();
}
uint64_t CachingBlockStore::estimateNumFreeBytes() const {
return _baseBlockStore->estimateNumFreeBytes();
}
void CachingBlockStore::release(unique_ref<Block> block) {
Key key = block->key();
_cache.push(key, std::move(block));
}
optional<unique_ref<Block>> CachingBlockStore::tryCreateInBaseStore(const Key &key, Data data) {
return _baseBlockStore->tryCreate(key, std::move(data));
}
void CachingBlockStore::removeFromBaseStore(cpputils::unique_ref<Block> block) {
_baseBlockStore->remove(std::move(block));
}
void CachingBlockStore::flush() {
_cache.flush();
}
uint64_t CachingBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
}
void CachingBlockStore::forEachBlock(std::function<void (const Key &)> callback) const {
_baseBlockStore->forEachBlock(callback);
for (NewBlock *newBlock : _newBlocks) {
callback(newBlock->key());
}
}
void CachingBlockStore::registerNewBlock(NewBlock *newBlock) {
unique_lock<mutex> lock(_newBlocksMutex);
_newBlocks.insert(newBlock);
}
void CachingBlockStore::unregisterNewBlock(NewBlock *newBlock) {
unique_lock<mutex> lock(_newBlocksMutex);
_newBlocks.erase(newBlock);
}
}
}

View File

@ -1,51 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
#include "cache/Cache.h"
#include "../../interface/BlockStore.h"
#include <unordered_set>
namespace blockstore {
namespace caching {
class NewBlock;
//TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel)
class CachingBlockStore final: public BlockStore {
public:
explicit CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
cpputils::unique_ref<Block> overwrite(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const Key &)> callback) const override;
void release(cpputils::unique_ref<Block> block);
boost::optional<cpputils::unique_ref<Block>> tryCreateInBaseStore(const Key &key, cpputils::Data data);
void removeFromBaseStore(cpputils::unique_ref<Block> block);
void registerNewBlock(NewBlock *newBlock);
void unregisterNewBlock(NewBlock *newBlock);
void flush();
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
std::unordered_set<NewBlock*> _newBlocks; // List of all new blocks that aren't in the base store yet.
Cache<Key, cpputils::unique_ref<Block>, 1000> _cache;
std::mutex _newBlocksMutex;
DISALLOW_COPY_AND_ASSIGN(CachingBlockStore);
};
}
}
#endif

View File

@ -153,5 +153,9 @@ void CachingBlockStore2::forEachBlock(std::function<void (const Key &)> callback
_baseBlockStore->forEachBlock(std::move(callback));
}
void CachingBlockStore2::flush() {
_cache.flush();
}
}
}

View File

@ -23,6 +23,8 @@ public:
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const Key &)> callback) const override;
void flush();
private:
// TODO Is a cache implementation with onEvict callback instead of destructor simpler?
class CachedBlock final {

View File

@ -1,76 +0,0 @@
#include "NewBlock.h"
#include "CachingBlockStore.h"
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
using cpputils::Data;
using boost::none;
namespace blockstore {
namespace caching {
NewBlock::NewBlock(const Key &key, Data data, CachingBlockStore *blockStore)
:Block(key),
_blockStore(blockStore),
_data(std::move(data)),
_baseBlock(none),
_dataChanged(true) {
_blockStore->registerNewBlock(this);
}
NewBlock::~NewBlock() {
writeToBaseBlockIfChanged();
}
const void *NewBlock::data() const {
return _data.data();
}
void NewBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area");
std::memcpy((uint8_t*)_data.data()+offset, source, size);
_dataChanged = true;
}
void NewBlock::writeToBaseBlockIfChanged() {
if (_dataChanged) {
if (_baseBlock == none) {
//TODO _data.copy() necessary?
auto newBase = _blockStore->tryCreateInBaseStore(key(), _data.copy());
ASSERT(newBase != boost::none, "Couldn't create base block"); //TODO What if tryCreate fails due to a duplicate key? We should ensure we don't use duplicate keys.
_blockStore->unregisterNewBlock(this); // block now exists in base store, we have to remove it from the list of blocks not in the base store.
_baseBlock = std::move(*newBase);
} else {
(*_baseBlock)->write(_data.data(), 0, _data.size());
}
_dataChanged = false;
}
}
void NewBlock::remove() {
if (_baseBlock != none) {
_blockStore->removeFromBaseStore(std::move(*_baseBlock));
} else {
// block doesn't exists in base store, we have to remove it from the list of blocks not in the base store.
_blockStore->unregisterNewBlock(this);
}
_dataChanged = false;
}
void NewBlock::flush() {
writeToBaseBlockIfChanged();
ASSERT(_baseBlock != none, "At this point, the base block should already have been created but wasn't");
(*_baseBlock)->flush();
}
size_t NewBlock::size() const {
return _data.size();
}
void NewBlock::resize(size_t newSize) {
_data = cpputils::DataUtils::resize(std::move(_data), newSize);
_dataChanged = true;
}
}
}

View File

@ -1,50 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
#include "../../interface/BlockStore.h"
#include <cpp-utils/data/Data.h>
#include <cpp-utils/macros.h>
#include <memory>
namespace blockstore {
namespace caching {
class CachingBlockStore;
//TODO Does it make sense to write a general DataBackedBlock that just stores a Data object and maps the block operations to it?
// Can we reuse that object somewhere else?
// Maybe a second abstract class for BlockRefBackedBlock?
// This is a block that was created in CachingBlockStore, but doesn't exist in the base block store yet.
// It only exists in the cache and it is created in the base block store when destructed.
class NewBlock final: public Block {
public:
NewBlock(const Key &key, cpputils::Data data, CachingBlockStore *blockStore);
~NewBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
void remove();
private:
CachingBlockStore *_blockStore;
cpputils::Data _data;
boost::optional<cpputils::unique_ref<Block>> _baseBlock;
bool _dataChanged;
void writeToBaseBlockIfChanged();
DISALLOW_COPY_AND_ASSIGN(NewBlock);
};
}
}
#endif

View File

@ -1,4 +1,4 @@
#include <blockstore/implementations/caching2/CachingBlockStore2.h>
#include <blockstore/implementations/caching/CachingBlockStore2.h>
#include <cpp-utils/crypto/symmetric/ciphers.h>
#include "parallelaccessfsblobstore/DirBlobRef.h"
#include "CryDevice.h"

View File

@ -19,10 +19,8 @@ set(SOURCES
implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp
implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp
implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp
implementations/caching2/CachingBlockStore2Test_Generic.cpp
implementations/caching2/CachingBlockStore2Test_Specific.cpp
implementations/caching/CachingBlockStoreTest_Generic.cpp
implementations/caching/CachingBlockStoreTest_Specific.cpp
implementations/caching/CachingBlockStore2Test_Generic.cpp
implementations/caching/CachingBlockStore2Test_Specific.cpp
implementations/caching/cache/QueueMapTest_Values.cpp
implementations/caching/cache/testutils/MinimalKeyType.cpp
implementations/caching/cache/testutils/CopyableMovableValueType.cpp

View File

@ -1,5 +1,5 @@
#include <blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h>
#include "blockstore/implementations/caching2/CachingBlockStore2.h"
#include "blockstore/implementations/caching/CachingBlockStore2.h"
#include "blockstore/implementations/inmemory/InMemoryBlockStore2.h"
#include "../../testutils/BlockStoreTest.h"
#include "../../testutils/BlockStore2Test.h"

View File

@ -1,6 +1,6 @@
#include <gtest/gtest.h>
#include "blockstore/implementations/caching/CachingBlockStore.h"
#include "blockstore/implementations/testfake/FakeBlockStore.h"
#include "blockstore/implementations/caching/CachingBlockStore2.h"
#include "blockstore/implementations/inmemory/InMemoryBlockStore2.h"
using ::testing::Test;
@ -8,24 +8,18 @@ using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using blockstore::testfake::FakeBlockStore;
using blockstore::inmemory::InMemoryBlockStore2;
using namespace blockstore::caching;
class CachingBlockStore2Test: public Test {
public:
CachingBlockStore2Test():
baseBlockStore(new FakeBlockStore),
blockStore(std::move(cpputils::nullcheck(std::unique_ptr<FakeBlockStore>(baseBlockStore)).value())) {
}
FakeBlockStore *baseBlockStore;
CachingBlockStore blockStore;
blockstore::Key CreateBlockReturnKey(const Data &initData) {
auto block = blockStore.create(initData);
block->flush();
return block->key();
baseBlockStore(new InMemoryBlockStore2),
blockStore(std::move(cpputils::nullcheck(std::unique_ptr<InMemoryBlockStore2>(baseBlockStore)).value())) {
}
InMemoryBlockStore2 *baseBlockStore;
CachingBlockStore2 blockStore;
};
TEST_F(CachingBlockStore2Test, PhysicalBlockSize_zerophysical) {
@ -33,15 +27,18 @@ TEST_F(CachingBlockStore2Test, PhysicalBlockSize_zerophysical) {
}
TEST_F(CachingBlockStore2Test, PhysicalBlockSize_zerovirtual) {
auto key = CreateBlockReturnKey(Data(0));
auto key = blockStore.create(Data(0));
blockStore.flush();
auto base = baseBlockStore->load(key).value();
EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(base->size()));
EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(base.size()));
}
TEST_F(CachingBlockStore2Test, PhysicalBlockSize_negativeboundaries) {
// This tests that a potential if/else in blockSizeFromPhysicalBlockSize that catches negative values has the
// correct boundary set. We test the highest value that is negative and the smallest value that is positive.
auto physicalSizeForVirtualSizeZero = baseBlockStore->load(CreateBlockReturnKey(Data(0))).value()->size();
auto key = blockStore.create(Data(0));
blockStore.flush();
auto physicalSizeForVirtualSizeZero = baseBlockStore->load(key).value().size();
if (physicalSizeForVirtualSizeZero > 0) {
EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero - 1));
}
@ -50,9 +47,10 @@ TEST_F(CachingBlockStore2Test, PhysicalBlockSize_negativeboundaries) {
}
TEST_F(CachingBlockStore2Test, PhysicalBlockSize_positive) {
auto key = CreateBlockReturnKey(Data(10*1024));
auto key = blockStore.create(Data(10*1024u));
blockStore.flush();
auto base = baseBlockStore->load(key).value();
EXPECT_EQ(10*1024u, blockStore.blockSizeFromPhysicalBlockSize(base->size()));
EXPECT_EQ(10*1024u, blockStore.blockSizeFromPhysicalBlockSize(base.size()));
}
// TODO Add test cases that flushing the block store doesn't destroy things (i.e. all test cases from BlockStoreTest, but with flushes inbetween)

View File

@ -1,23 +0,0 @@
#include "blockstore/implementations/caching/CachingBlockStore.h"
#include "blockstore/implementations/testfake/FakeBlockStore.h"
#include "../../testutils/BlockStoreTest.h"
#include <gtest/gtest.h>
#include <cpp-utils/pointer/unique_ref_boost_optional_gtest_workaround.h>
using blockstore::BlockStore;
using blockstore::caching::CachingBlockStore;
using blockstore::testfake::FakeBlockStore;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
class CachingBlockStoreTestFixture: public BlockStoreTestFixture {
public:
unique_ref<BlockStore> createBlockStore() override {
return make_unique_ref<CachingBlockStore>(make_unique_ref<FakeBlockStore>());
}
};
INSTANTIATE_TYPED_TEST_CASE_P(Caching, BlockStoreTest, CachingBlockStoreTestFixture);
//TODO Add specific tests for the blockstore

View File

@ -1,56 +0,0 @@
#include <gtest/gtest.h>
#include "blockstore/implementations/caching/CachingBlockStore.h"
#include "blockstore/implementations/testfake/FakeBlockStore.h"
using ::testing::Test;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using blockstore::testfake::FakeBlockStore;
using namespace blockstore::caching;
class CachingBlockStoreTest: public Test {
public:
CachingBlockStoreTest():
baseBlockStore(new FakeBlockStore),
blockStore(std::move(cpputils::nullcheck(std::unique_ptr<FakeBlockStore>(baseBlockStore)).value())) {
}
FakeBlockStore *baseBlockStore;
CachingBlockStore blockStore;
blockstore::Key CreateBlockReturnKey(const Data &initData) {
auto block = blockStore.create(initData);
block->flush();
return block->key();
}
};
TEST_F(CachingBlockStoreTest, PhysicalBlockSize_zerophysical) {
EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(0));
}
TEST_F(CachingBlockStoreTest, PhysicalBlockSize_zerovirtual) {
auto key = CreateBlockReturnKey(Data(0));
auto base = baseBlockStore->load(key).value();
EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(base->size()));
}
TEST_F(CachingBlockStoreTest, PhysicalBlockSize_negativeboundaries) {
// This tests that a potential if/else in blockSizeFromPhysicalBlockSize that catches negative values has the
// correct boundary set. We test the highest value that is negative and the smallest value that is positive.
auto physicalSizeForVirtualSizeZero = baseBlockStore->load(CreateBlockReturnKey(Data(0))).value()->size();
if (physicalSizeForVirtualSizeZero > 0) {
EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero - 1));
}
EXPECT_EQ(0u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero));
EXPECT_EQ(1u, blockStore.blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero + 1));
}
TEST_F(CachingBlockStoreTest, PhysicalBlockSize_positive) {
auto key = CreateBlockReturnKey(Data(10*1024));
auto base = baseBlockStore->load(key).value();
EXPECT_EQ(10*1024u, blockStore.blockSizeFromPhysicalBlockSize(base->size()));
}