A block stores its key

This commit is contained in:
Sebastian Messmer 2015-01-24 22:08:41 +01:00
parent f4398dfeec
commit 196b543cbb
15 changed files with 78 additions and 45 deletions

View File

@ -14,13 +14,13 @@ using std::ios;
namespace blockstore { namespace blockstore {
namespace inmemory { namespace inmemory {
InMemoryBlock::InMemoryBlock(size_t size) InMemoryBlock::InMemoryBlock(const Key &key, size_t size)
: _data(make_shared<Data>(size)) { : Block(key), _data(make_shared<Data>(size)) {
_data->FillWithZeroes(); _data->FillWithZeroes();
} }
InMemoryBlock::InMemoryBlock(const InMemoryBlock &rhs) InMemoryBlock::InMemoryBlock(const InMemoryBlock &rhs)
: _data(rhs._data) { : Block(rhs), _data(rhs._data) {
} }
InMemoryBlock::~InMemoryBlock() { InMemoryBlock::~InMemoryBlock() {

View File

@ -11,7 +11,7 @@ class InMemoryBlockStore;
class InMemoryBlock: public Block { class InMemoryBlock: public Block {
public: public:
InMemoryBlock(size_t size); InMemoryBlock(const Key &key, size_t size);
InMemoryBlock(const InMemoryBlock &rhs); InMemoryBlock(const InMemoryBlock &rhs);
virtual ~InMemoryBlock(); virtual ~InMemoryBlock();

View File

@ -6,6 +6,8 @@ using std::make_unique;
using std::string; using std::string;
using std::mutex; using std::mutex;
using std::lock_guard; using std::lock_guard;
using std::piecewise_construct;
using std::make_tuple;
namespace blockstore { namespace blockstore {
namespace inmemory { namespace inmemory {
@ -14,7 +16,7 @@ InMemoryBlockStore::InMemoryBlockStore()
: _blocks() {} : _blocks() {}
unique_ptr<Block> InMemoryBlockStore::create(const Key &key, size_t size) { unique_ptr<Block> InMemoryBlockStore::create(const Key &key, size_t size) {
auto insert_result = _blocks.emplace(key.ToString(), size); auto insert_result = _blocks.emplace(piecewise_construct, make_tuple(key.ToString()), make_tuple(key, size));
if (!insert_result.second) { if (!insert_result.second) {
return nullptr; return nullptr;

View File

@ -19,12 +19,12 @@ namespace bf = boost::filesystem;
namespace blockstore { namespace blockstore {
namespace ondisk { namespace ondisk {
OnDiskBlock::OnDiskBlock(const bf::path &filepath, size_t size) OnDiskBlock::OnDiskBlock(const Key &key, const bf::path &filepath, size_t size)
: _filepath(filepath), _data(size) { : Block(key), _filepath(filepath), _data(size) {
} }
OnDiskBlock::OnDiskBlock(const bf::path &filepath, Data &&data) OnDiskBlock::OnDiskBlock(const Key &key, const bf::path &filepath, Data &&data)
: _filepath(filepath), _data(std::move(data)) { : Block(key), _filepath(filepath), _data(std::move(data)) {
} }
OnDiskBlock::~OnDiskBlock() { OnDiskBlock::~OnDiskBlock() {
@ -43,7 +43,8 @@ size_t OnDiskBlock::size() const {
return _data.size(); return _data.size();
} }
unique_ptr<OnDiskBlock> OnDiskBlock::LoadFromDisk(const bf::path &filepath) { unique_ptr<OnDiskBlock> OnDiskBlock::LoadFromDisk(const bf::path &rootdir, const Key &key) {
auto filepath = rootdir / key.ToString();
try { try {
//If it isn't a file, Data::LoadFromFile() would usually also crash. We still need this extra check //If it isn't a file, Data::LoadFromFile() would usually also crash. We still need this extra check
//upfront, because Data::LoadFromFile() doesn't crash if we give it the path of a directory //upfront, because Data::LoadFromFile() doesn't crash if we give it the path of a directory
@ -52,18 +53,19 @@ unique_ptr<OnDiskBlock> OnDiskBlock::LoadFromDisk(const bf::path &filepath) {
return nullptr; return nullptr;
} }
Data data = Data::LoadFromFile(filepath); Data data = Data::LoadFromFile(filepath);
return unique_ptr<OnDiskBlock>(new OnDiskBlock(filepath, std::move(data))); return unique_ptr<OnDiskBlock>(new OnDiskBlock(key, filepath, std::move(data)));
} catch (const FileDoesntExistException &e) { } catch (const FileDoesntExistException &e) {
return nullptr; return nullptr;
} }
} }
unique_ptr<OnDiskBlock> OnDiskBlock::CreateOnDisk(const bf::path &filepath, size_t size) { unique_ptr<OnDiskBlock> OnDiskBlock::CreateOnDisk(const bf::path &rootdir, const Key &key, size_t size) {
auto filepath = rootdir / key.ToString();
if (bf::exists(filepath)) { if (bf::exists(filepath)) {
return nullptr; return nullptr;
} }
auto block = unique_ptr<OnDiskBlock>(new OnDiskBlock(filepath, size)); auto block = unique_ptr<OnDiskBlock>(new OnDiskBlock(key, filepath, size));
block->_fillDataWithZeroes(); block->_fillDataWithZeroes();
block->_storeToDisk(); block->_storeToDisk();
return block; return block;

View File

@ -17,8 +17,8 @@ class OnDiskBlock: public Block {
public: public:
virtual ~OnDiskBlock(); virtual ~OnDiskBlock();
static std::unique_ptr<OnDiskBlock> LoadFromDisk(const boost::filesystem::path &filepath); static std::unique_ptr<OnDiskBlock> LoadFromDisk(const boost::filesystem::path &rootdir, const Key &key);
static std::unique_ptr<OnDiskBlock> CreateOnDisk(const boost::filesystem::path &filepath, size_t size); static std::unique_ptr<OnDiskBlock> CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, size_t size);
void *data() override; void *data() override;
const void *data() const override; const void *data() const override;
@ -31,8 +31,8 @@ private:
const boost::filesystem::path _filepath; const boost::filesystem::path _filepath;
Data _data; Data _data;
OnDiskBlock(const boost::filesystem::path &filepath, size_t size); OnDiskBlock(const Key &key, const boost::filesystem::path &filepath, size_t size);
OnDiskBlock(const boost::filesystem::path &filepath, Data &&data); OnDiskBlock(const Key &key, const boost::filesystem::path &filepath, Data &&data);
void _fillDataWithZeroes(); void _fillDataWithZeroes();
void _storeToDisk() const; void _storeToDisk() const;

View File

@ -16,8 +16,7 @@ OnDiskBlockStore::OnDiskBlockStore(const boost::filesystem::path &rootdir)
: _rootdir(rootdir) {} : _rootdir(rootdir) {}
unique_ptr<Block> OnDiskBlockStore::create(const Key &key, size_t size) { unique_ptr<Block> OnDiskBlockStore::create(const Key &key, size_t size) {
auto file_path = _rootdir / key.ToString(); auto block = OnDiskBlock::CreateOnDisk(_rootdir, key, size);
auto block = OnDiskBlock::CreateOnDisk(file_path, size);
if (!block) { if (!block) {
return nullptr; return nullptr;
@ -26,8 +25,7 @@ unique_ptr<Block> OnDiskBlockStore::create(const Key &key, size_t size) {
} }
unique_ptr<Block> OnDiskBlockStore::load(const Key &key) { unique_ptr<Block> OnDiskBlockStore::load(const Key &key) {
auto file_path = _rootdir / key.ToString(); return OnDiskBlock::LoadFromDisk(_rootdir, key);
return OnDiskBlock::LoadFromDisk(file_path);
} }
} }

View File

@ -14,8 +14,8 @@ using std::string;
namespace blockstore { namespace blockstore {
namespace testfake { namespace testfake {
FakeBlock::FakeBlock(FakeBlockStore *store, const string &key, shared_ptr<Data> data) FakeBlock::FakeBlock(FakeBlockStore *store, const Key &key, shared_ptr<Data> data)
: _store(store), _key(key), _data(data) { : Block(key), _store(store), _data(data) {
} }
FakeBlock::~FakeBlock() { FakeBlock::~FakeBlock() {
@ -35,7 +35,7 @@ size_t FakeBlock::size() const {
} }
void FakeBlock::flush() { void FakeBlock::flush() {
_store->updateData(_key, *_data); _store->updateData(key(), *_data);
} }
} }

View File

@ -13,7 +13,7 @@ class FakeBlockStore;
class FakeBlock: public Block { class FakeBlock: public Block {
public: public:
FakeBlock(FakeBlockStore *store, const std::string &key, std::shared_ptr<Data> data); FakeBlock(FakeBlockStore *store, const Key &key, std::shared_ptr<Data> data);
virtual ~FakeBlock(); virtual ~FakeBlock();
void *data() override; void *data() override;
@ -25,7 +25,6 @@ public:
private: private:
FakeBlockStore *_store; FakeBlockStore *_store;
const std::string _key;
std::shared_ptr<Data> _data; std::shared_ptr<Data> _data;
DISALLOW_COPY_AND_ASSIGN(FakeBlock); DISALLOW_COPY_AND_ASSIGN(FakeBlock);

View File

@ -30,20 +30,20 @@ unique_ptr<Block> FakeBlockStore::load(const Key &key) {
//Return a copy of the stored data //Return a copy of the stored data
string key_string = key.ToString(); string key_string = key.ToString();
try { try {
return makeFakeBlockFromData(key_string, _blocks.at(key_string)); return makeFakeBlockFromData(key, _blocks.at(key_string));
} catch (const std::out_of_range &e) { } catch (const std::out_of_range &e) {
return nullptr; return nullptr;
} }
} }
unique_ptr<Block> FakeBlockStore::makeFakeBlockFromData(const std::string &key, const Data &data) { unique_ptr<Block> FakeBlockStore::makeFakeBlockFromData(const Key &key, const Data &data) {
auto newdata = make_shared<Data>(data.copy()); auto newdata = make_shared<Data>(data.copy());
_used_dataregions_for_blocks.push_back(newdata); _used_dataregions_for_blocks.push_back(newdata);
return make_unique<FakeBlock>(this, key, newdata); return make_unique<FakeBlock>(this, key, newdata);
} }
void FakeBlockStore::updateData(const std::string &key, const Data &data) { void FakeBlockStore::updateData(const Key &key, const Data &data) {
Data &stored_data = _blocks.at(key); Data &stored_data = _blocks.at(key.ToString());
assert(data.size() == stored_data.size()); assert(data.size() == stored_data.size());
std::memcpy(stored_data.data(), data.data(), data.size()); std::memcpy(stored_data.data(), data.data(), data.size());
} }

View File

@ -34,7 +34,7 @@ public:
std::unique_ptr<Block> create(const Key &key, size_t size) override; std::unique_ptr<Block> create(const Key &key, size_t size) override;
std::unique_ptr<Block> load(const Key &key) override; std::unique_ptr<Block> load(const Key &key) override;
void updateData(const std::string &key, const Data &data); void updateData(const Key &key, const Data &data);
private: private:
std::map<std::string, Data> _blocks; std::map<std::string, Data> _blocks;
@ -46,7 +46,7 @@ private:
//We want to avoid this for the reasons mentioned above (overflow data). //We want to avoid this for the reasons mentioned above (overflow data).
std::vector<std::shared_ptr<Data>> _used_dataregions_for_blocks; std::vector<std::shared_ptr<Data>> _used_dataregions_for_blocks;
std::unique_ptr<Block> makeFakeBlockFromData(const std::string &key, const Data &data); std::unique_ptr<Block> makeFakeBlockFromData(const Key &key, const Data &data);
DISALLOW_COPY_AND_ASSIGN(FakeBlockStore); DISALLOW_COPY_AND_ASSIGN(FakeBlockStore);
}; };

View File

@ -3,6 +3,7 @@
#define BLOCKSTORE_INTERFACE_BLOCK_H_ #define BLOCKSTORE_INTERFACE_BLOCK_H_
#include <cstring> #include <cstring>
#include "blockstore/utils/Key.h"
namespace blockstore { namespace blockstore {
@ -16,6 +17,16 @@ public:
virtual void flush() = 0; virtual void flush() = 0;
virtual size_t size() const = 0; virtual size_t size() const = 0;
const Key &key() const {
return _key;
}
protected:
Block(const Key &key) : _key(key) {}
private:
const Key _key;
}; };
} }

View File

@ -3,7 +3,7 @@
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "test/testutils/TempFile.h" #include "test/testutils/TempFile.h"
#include "test/testutils/TempDir.h"
using ::testing::Test; using ::testing::Test;
using ::testing::WithParamInterface; using ::testing::WithParamInterface;
@ -20,23 +20,27 @@ class OnDiskBlockCreateTest: public Test {
public: public:
OnDiskBlockCreateTest() OnDiskBlockCreateTest()
// Don't create the temp file yet (therefore pass false to the TempFile constructor) // Don't create the temp file yet (therefore pass false to the TempFile constructor)
: file(false) { : dir(),
key(Key::FromString("1491BB4932A389EE14BC7090AC772972")),
file(dir.path() / key.ToString(), false) {
} }
TempDir dir;
Key key;
TempFile file; TempFile file;
}; };
TEST_F(OnDiskBlockCreateTest, CreatingBlockCreatesFile) { TEST_F(OnDiskBlockCreateTest, CreatingBlockCreatesFile) {
EXPECT_FALSE(bf::exists(file.path())); EXPECT_FALSE(bf::exists(file.path()));
auto block = OnDiskBlock::CreateOnDisk(file.path(), 0); auto block = OnDiskBlock::CreateOnDisk(dir.path(), key, 0);
EXPECT_TRUE(bf::exists(file.path())); EXPECT_TRUE(bf::exists(file.path()));
EXPECT_TRUE(bf::is_regular_file(file.path())); EXPECT_TRUE(bf::is_regular_file(file.path()));
} }
TEST_F(OnDiskBlockCreateTest, CreatingExistingBlockReturnsNull) { TEST_F(OnDiskBlockCreateTest, CreatingExistingBlockReturnsNull) {
auto block1 = OnDiskBlock::CreateOnDisk(file.path(), 0); auto block1 = OnDiskBlock::CreateOnDisk(dir.path(), key, 0);
auto block2 = OnDiskBlock::CreateOnDisk(file.path(), 0); auto block2 = OnDiskBlock::CreateOnDisk(dir.path(), key, 0);
EXPECT_TRUE((bool)block1); EXPECT_TRUE((bool)block1);
EXPECT_FALSE((bool)block2); EXPECT_FALSE((bool)block2);
} }
@ -47,7 +51,7 @@ public:
Data ZEROES; Data ZEROES;
OnDiskBlockCreateSizeTest(): OnDiskBlockCreateSizeTest():
block(OnDiskBlock::CreateOnDisk(file.path(), GetParam())), block(OnDiskBlock::CreateOnDisk(dir.path(), key, GetParam())),
ZEROES(block->size()) ZEROES(block->size())
{ {
ZEROES.FillWithZeroes(); ZEROES.FillWithZeroes();
@ -56,6 +60,7 @@ public:
INSTANTIATE_TEST_CASE_P(OnDiskBlockCreateSizeTest, OnDiskBlockCreateSizeTest, Values(0, 1, 5, 1024, 10*1024*1024)); INSTANTIATE_TEST_CASE_P(OnDiskBlockCreateSizeTest, OnDiskBlockCreateSizeTest, Values(0, 1, 5, 1024, 10*1024*1024));
TEST_P(OnDiskBlockCreateSizeTest, OnDiskSizeIsCorrect) { TEST_P(OnDiskBlockCreateSizeTest, OnDiskSizeIsCorrect) {
printf((std::string()+file.path().c_str()+"\n").c_str());
Data fileContent = Data::LoadFromFile(file.path()); Data fileContent = Data::LoadFromFile(file.path());
EXPECT_EQ(GetParam(), fileContent.size()); EXPECT_EQ(GetParam(), fileContent.size());
} }

View File

@ -4,6 +4,7 @@
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "test/testutils/TempFile.h" #include "test/testutils/TempFile.h"
#include "test/testutils/TempDir.h"
using ::testing::Test; using ::testing::Test;
using ::testing::WithParamInterface; using ::testing::WithParamInterface;
@ -20,21 +21,26 @@ class OnDiskBlockFlushTest: public Test, public WithParamInterface<size_t> {
public: public:
OnDiskBlockFlushTest() OnDiskBlockFlushTest()
// Don't create the temp file yet (therefore pass false to the TempFile constructor) // Don't create the temp file yet (therefore pass false to the TempFile constructor)
: file(false), randomData(GetParam()) { : dir(),
key(Key::FromString("1491BB4932A389EE14BC7090AC772972")),
file(dir.path() / key.ToString(), false),
randomData(GetParam()) {
} }
TempDir dir;
Key key;
TempFile file; TempFile file;
DataBlockFixture randomData; DataBlockFixture randomData;
unique_ptr<OnDiskBlock> CreateBlockAndLoadItFromDisk() { unique_ptr<OnDiskBlock> CreateBlockAndLoadItFromDisk() {
{ {
auto block = OnDiskBlock::CreateOnDisk(file.path(), randomData.size()); auto block = OnDiskBlock::CreateOnDisk(dir.path(), key, randomData.size());
} }
return OnDiskBlock::LoadFromDisk(file.path()); return OnDiskBlock::LoadFromDisk(dir.path(), key);
} }
unique_ptr<OnDiskBlock> CreateBlock() { unique_ptr<OnDiskBlock> CreateBlock() {
return OnDiskBlock::CreateOnDisk(file.path(), randomData.size()); return OnDiskBlock::CreateOnDisk(dir.path(), key, randomData.size());
} }
void WriteDataToBlock(const unique_ptr<OnDiskBlock> &block) { void WriteDataToBlock(const unique_ptr<OnDiskBlock> &block) {

View File

@ -5,6 +5,7 @@
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "test/testutils/TempFile.h" #include "test/testutils/TempFile.h"
#include "test/testutils/TempDir.h"
#include <fstream> #include <fstream>
using ::testing::Test; using ::testing::Test;
@ -22,6 +23,13 @@ namespace bf = boost::filesystem;
class OnDiskBlockLoadTest: public Test, public WithParamInterface<size_t> { class OnDiskBlockLoadTest: public Test, public WithParamInterface<size_t> {
public: public:
OnDiskBlockLoadTest():
dir(),
key(Key::FromString("1491BB4932A389EE14BC7090AC772972")),
file(dir.path() / key.ToString()) {
}
TempDir dir;
Key key;
TempFile file; TempFile file;
void SetFileSize(size_t size) { void SetFileSize(size_t size) {
@ -37,7 +45,7 @@ public:
} }
unique_ptr<OnDiskBlock> LoadBlock() { unique_ptr<OnDiskBlock> LoadBlock() {
return OnDiskBlock::LoadFromDisk(file.path()); return OnDiskBlock::LoadFromDisk(dir.path(), key);
} }
void EXPECT_BLOCK_DATA_EQ(const DataBlockFixture &expected, const OnDiskBlock &actual) { void EXPECT_BLOCK_DATA_EQ(const DataBlockFixture &expected, const OnDiskBlock &actual) {
@ -65,8 +73,8 @@ TEST_P(OnDiskBlockLoadTest, LoadedDataIsCorrect) {
} }
TEST_F(OnDiskBlockLoadTest, LoadNotExistingBlock) { TEST_F(OnDiskBlockLoadTest, LoadNotExistingBlock) {
TempFile file2(false); // Pass false, so the file isn't created. Key key2 = Key::FromString("272EE5517627CFA147A971A8E6E747E0");
EXPECT_FALSE( EXPECT_FALSE(
(bool)OnDiskBlock::LoadFromDisk(file2.path()) (bool)OnDiskBlock::LoadFromDisk(dir.path(), key2)
); );
} }

View File

@ -28,10 +28,12 @@ public:
class BlockMock: public Block { class BlockMock: public Block {
public: public:
BlockMock(): Block(Key::CreateRandomKey()) {}
MOCK_METHOD0(data, void*()); MOCK_METHOD0(data, void*());
MOCK_CONST_METHOD0(data, const void*()); MOCK_CONST_METHOD0(data, const void*());
MOCK_METHOD0(flush, void()); MOCK_METHOD0(flush, void());
MOCK_CONST_METHOD0(size, size_t()); MOCK_CONST_METHOD0(size, size_t());
MOCK_CONST_METHOD0(key, const Key&());
}; };
class BlockStoreWithRandomKeysTest: public Test { class BlockStoreWithRandomKeysTest: public Test {