From 868c06a5f1156ba2990c74e2f4bde73ceb29c916 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Tue, 9 Dec 2014 17:19:59 +0100 Subject: [PATCH] Rename blobstore to blockstore --- src/blockstore/CMakeLists.txt | 3 + src/blockstore/implementations/CMakeLists.txt | 2 + .../implementations/inmemory/CMakeLists.txt | 3 + .../inmemory/InMemoryBlock.cpp | 45 ++++ .../implementations/inmemory/InMemoryBlock.h | 32 +++ .../inmemory/InMemoryBlockStore.cpp | 38 +++ .../inmemory/InMemoryBlockStore.h | 31 +++ .../implementations/ondisk/CMakeLists.txt | 3 + .../ondisk/FileAlreadyExistsException.cpp | 19 ++ .../ondisk/FileAlreadyExistsException.h | 21 ++ .../implementations/ondisk/OnDiskBlock.cpp | 85 +++++++ .../implementations/ondisk/OnDiskBlock.h | 46 ++++ .../ondisk/OnDiskBlockStore.cpp | 35 +++ .../implementations/ondisk/OnDiskBlockStore.h | 31 +++ src/blockstore/interface/Block.h | 24 ++ src/blockstore/interface/BlockStore.h | 29 +++ src/blockstore/interface/CMakeLists.txt | 1 + .../helpers/BlockStoreWithRandomKeys.cpp | 19 ++ .../helpers/BlockStoreWithRandomKeys.h | 27 +++ src/blockstore/utils/BlockWithKey.h | 20 ++ src/blockstore/utils/CMakeLists.txt | 3 + src/blockstore/utils/Data.cpp | 91 +++++++ src/blockstore/utils/Data.h | 43 ++++ .../utils/FileDoesntExistException.cpp | 17 ++ .../utils/FileDoesntExistException.h | 19 ++ src/blockstore/utils/RandomKeyGenerator.cpp | 53 ++++ src/blockstore/utils/RandomKeyGenerator.h | 37 +++ .../inmemory/InMemoryBlockStoreTest.cpp | 31 +++ .../ondisk/OnDiskBlockStoreTest.cpp | 35 +++ .../OnDiskBlockTest/OnDiskBlockCreateTest.cpp | 78 ++++++ .../OnDiskBlockTest/OnDiskBlockFlushTest.cpp | 111 +++++++++ .../OnDiskBlockTest/OnDiskBlockLoadTest.cpp | 72 ++++++ .../blockstore/interface/BlockStoreTest.cpp | 4 + src/test/blockstore/interface/BlockTest.cpp | 4 + .../helpers/BlockStoreWithRandomKeysTest.cpp | 117 +++++++++ .../blockstore/testutils/BlockStoreTest.h | 226 ++++++++++++++++++ .../testutils/BlockStoreWithRandomKeysTest.h | 81 +++++++ src/test/blockstore/utils/DataTest.cpp | 171 +++++++++++++ .../utils/RandomKeyGeneratorTest.cpp | 20 ++ 39 files changed, 1727 insertions(+) create mode 100644 src/blockstore/CMakeLists.txt create mode 100644 src/blockstore/implementations/CMakeLists.txt create mode 100644 src/blockstore/implementations/inmemory/CMakeLists.txt create mode 100644 src/blockstore/implementations/inmemory/InMemoryBlock.cpp create mode 100644 src/blockstore/implementations/inmemory/InMemoryBlock.h create mode 100644 src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp create mode 100644 src/blockstore/implementations/inmemory/InMemoryBlockStore.h create mode 100644 src/blockstore/implementations/ondisk/CMakeLists.txt create mode 100644 src/blockstore/implementations/ondisk/FileAlreadyExistsException.cpp create mode 100644 src/blockstore/implementations/ondisk/FileAlreadyExistsException.h create mode 100644 src/blockstore/implementations/ondisk/OnDiskBlock.cpp create mode 100644 src/blockstore/implementations/ondisk/OnDiskBlock.h create mode 100644 src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp create mode 100644 src/blockstore/implementations/ondisk/OnDiskBlockStore.h create mode 100644 src/blockstore/interface/Block.h create mode 100644 src/blockstore/interface/BlockStore.h create mode 100644 src/blockstore/interface/CMakeLists.txt create mode 100644 src/blockstore/interface/helpers/BlockStoreWithRandomKeys.cpp create mode 100644 src/blockstore/interface/helpers/BlockStoreWithRandomKeys.h create mode 100644 src/blockstore/utils/BlockWithKey.h create mode 100644 src/blockstore/utils/CMakeLists.txt create mode 100644 src/blockstore/utils/Data.cpp create mode 100644 src/blockstore/utils/Data.h create mode 100644 src/blockstore/utils/FileDoesntExistException.cpp create mode 100644 src/blockstore/utils/FileDoesntExistException.h create mode 100644 src/blockstore/utils/RandomKeyGenerator.cpp create mode 100644 src/blockstore/utils/RandomKeyGenerator.h create mode 100644 src/test/blockstore/implementations/inmemory/InMemoryBlockStoreTest.cpp create mode 100644 src/test/blockstore/implementations/ondisk/OnDiskBlockStoreTest.cpp create mode 100644 src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp create mode 100644 src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp create mode 100644 src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp create mode 100644 src/test/blockstore/interface/BlockStoreTest.cpp create mode 100644 src/test/blockstore/interface/BlockTest.cpp create mode 100644 src/test/blockstore/interface/helpers/BlockStoreWithRandomKeysTest.cpp create mode 100644 src/test/blockstore/testutils/BlockStoreTest.h create mode 100644 src/test/blockstore/testutils/BlockStoreWithRandomKeysTest.h create mode 100644 src/test/blockstore/utils/DataTest.cpp create mode 100644 src/test/blockstore/utils/RandomKeyGeneratorTest.cpp diff --git a/src/blockstore/CMakeLists.txt b/src/blockstore/CMakeLists.txt new file mode 100644 index 00000000..d4f05ddd --- /dev/null +++ b/src/blockstore/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(interface) +add_subdirectory(utils) +add_subdirectory(implementations) diff --git a/src/blockstore/implementations/CMakeLists.txt b/src/blockstore/implementations/CMakeLists.txt new file mode 100644 index 00000000..22d8c52d --- /dev/null +++ b/src/blockstore/implementations/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(ondisk) +add_subdirectory(inmemory) diff --git a/src/blockstore/implementations/inmemory/CMakeLists.txt b/src/blockstore/implementations/inmemory/CMakeLists.txt new file mode 100644 index 00000000..3b66b10d --- /dev/null +++ b/src/blockstore/implementations/inmemory/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(blockstore_inmemory InMemoryBlock.cpp InMemoryBlockStore.cpp) + +target_link_libraries(blockstore_inmemory blockstore_interface blockstore_utils) diff --git a/src/blockstore/implementations/inmemory/InMemoryBlock.cpp b/src/blockstore/implementations/inmemory/InMemoryBlock.cpp new file mode 100644 index 00000000..f4d773dd --- /dev/null +++ b/src/blockstore/implementations/inmemory/InMemoryBlock.cpp @@ -0,0 +1,45 @@ +#include +#include +#include + +using std::unique_ptr; +using std::make_unique; +using std::make_shared; +using std::istream; +using std::ostream; +using std::ifstream; +using std::ofstream; +using std::ios; + +namespace blockstore { +namespace inmemory { + +InMemoryBlock::InMemoryBlock(size_t size) + : _data(make_shared(size)) { + _data->FillWithZeroes(); +} + +InMemoryBlock::InMemoryBlock(const InMemoryBlock &rhs) + : _data(rhs._data) { +} + +InMemoryBlock::~InMemoryBlock() { +} + +void *InMemoryBlock::data() { + return _data->data(); +} + +const void *InMemoryBlock::data() const { + return _data->data(); +} + +size_t InMemoryBlock::size() const { + return _data->size(); +} + +void InMemoryBlock::flush() { +} + +} +} diff --git a/src/blockstore/implementations/inmemory/InMemoryBlock.h b/src/blockstore/implementations/inmemory/InMemoryBlock.h new file mode 100644 index 00000000..68935f64 --- /dev/null +++ b/src/blockstore/implementations/inmemory/InMemoryBlock.h @@ -0,0 +1,32 @@ +#pragma once +#ifndef BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_ +#define BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_ + +#include +#include + +namespace blockstore { +namespace inmemory { +class InMemoryBlockStore; + +class InMemoryBlock: public Block { +public: + InMemoryBlock(size_t size); + InMemoryBlock(const InMemoryBlock &rhs); + virtual ~InMemoryBlock(); + + void *data() override; + const void *data() const override; + + void flush() override; + + size_t size() const override; + +private: + std::shared_ptr _data; +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp b/src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp new file mode 100644 index 00000000..d8bfdab8 --- /dev/null +++ b/src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp @@ -0,0 +1,38 @@ +#include +#include +#include + +using std::unique_ptr; +using std::make_unique; +using std::string; +using std::mutex; +using std::lock_guard; + +namespace blockstore { +namespace inmemory { + +InMemoryBlockStore::InMemoryBlockStore() + : _blocks() {} + +unique_ptr InMemoryBlockStore::create(const std::string &key, size_t size) { + auto insert_result = _blocks.emplace(key, size); + + if (!insert_result.second) { + return nullptr; + } + + //Return a copy of the stored InMemoryBlock + return make_unique(key, make_unique(insert_result.first->second)); +} + +unique_ptr InMemoryBlockStore::load(const string &key) { + //Return a copy of the stored InMemoryBlock + try { + return make_unique(_blocks.at(key)); + } catch (const std::out_of_range &e) { + return nullptr; + } +} + +} +} diff --git a/src/blockstore/implementations/inmemory/InMemoryBlockStore.h b/src/blockstore/implementations/inmemory/InMemoryBlockStore.h new file mode 100644 index 00000000..67b4392d --- /dev/null +++ b/src/blockstore/implementations/inmemory/InMemoryBlockStore.h @@ -0,0 +1,31 @@ +#pragma once +#ifndef BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_ +#define BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_ + +#include +#include "fspp/utils/macros.h" + +#include +#include + +namespace blockstore { +namespace inmemory { +class InMemoryBlock; + +class InMemoryBlockStore: public BlockStoreWithRandomKeys { +public: + InMemoryBlockStore(); + + std::unique_ptr create(const std::string &key, size_t size) override; + std::unique_ptr load(const std::string &key) override; + +private: + std::map _blocks; + + DISALLOW_COPY_AND_ASSIGN(InMemoryBlockStore); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/ondisk/CMakeLists.txt b/src/blockstore/implementations/ondisk/CMakeLists.txt new file mode 100644 index 00000000..f907eb78 --- /dev/null +++ b/src/blockstore/implementations/ondisk/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(blockstore_ondisk OnDiskBlock.cpp OnDiskBlockStore.cpp FileAlreadyExistsException.cpp) + +target_link_libraries(blockstore_ondisk blockstore_interface blockstore_utils boost_filesystem boost_system) diff --git a/src/blockstore/implementations/ondisk/FileAlreadyExistsException.cpp b/src/blockstore/implementations/ondisk/FileAlreadyExistsException.cpp new file mode 100644 index 00000000..70aa369e --- /dev/null +++ b/src/blockstore/implementations/ondisk/FileAlreadyExistsException.cpp @@ -0,0 +1,19 @@ +#include + +namespace bf = boost::filesystem; + +using std::runtime_error; +using std::string; + +namespace blockstore { +namespace ondisk { + +FileAlreadyExistsException::FileAlreadyExistsException(const bf::path &filepath) +: runtime_error(string("The file ")+filepath.c_str()+" already exists") { +} + +FileAlreadyExistsException::~FileAlreadyExistsException() { +} + +} +} diff --git a/src/blockstore/implementations/ondisk/FileAlreadyExistsException.h b/src/blockstore/implementations/ondisk/FileAlreadyExistsException.h new file mode 100644 index 00000000..2a23fdd1 --- /dev/null +++ b/src/blockstore/implementations/ondisk/FileAlreadyExistsException.h @@ -0,0 +1,21 @@ +#pragma once +#ifndef BLOCKSTORE_IMPLEMENTATIONS_ONDISK_FILEALREADYEXISTSEXCEPTION_H_ +#define BLOCKSTORE_IMPLEMENTATIONS_ONDISK_FILEALREADYEXISTSEXCEPTION_H_ + +#include + +#include + +namespace blockstore { +namespace ondisk { + +class FileAlreadyExistsException: public std::runtime_error { +public: + FileAlreadyExistsException(const boost::filesystem::path &filepath); + virtual ~FileAlreadyExistsException(); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/ondisk/OnDiskBlock.cpp b/src/blockstore/implementations/ondisk/OnDiskBlock.cpp new file mode 100644 index 00000000..58139b4f --- /dev/null +++ b/src/blockstore/implementations/ondisk/OnDiskBlock.cpp @@ -0,0 +1,85 @@ +#include +#include +#include +#include +#include +#include +#include + +using std::unique_ptr; +using std::make_unique; +using std::istream; +using std::ostream; +using std::ifstream; +using std::ofstream; +using std::ios; + +namespace bf = boost::filesystem; + +namespace blockstore { +namespace ondisk { + +OnDiskBlock::OnDiskBlock(const bf::path &filepath, size_t size) + : _filepath(filepath), _data(size) { +} + +OnDiskBlock::OnDiskBlock(const bf::path &filepath, Data &&data) + : _filepath(filepath), _data(std::move(data)) { +} + +OnDiskBlock::~OnDiskBlock() { + _storeToDisk(); +} + +void *OnDiskBlock::data() { + return _data.data(); +} + +const void *OnDiskBlock::data() const { + return _data.data(); +} + +size_t OnDiskBlock::size() const { + return _data.size(); +} + +unique_ptr OnDiskBlock::LoadFromDisk(const bf::path &filepath) { + try { + //If it isn't a file, Data::LoadFromFile() would usually also crash. We still need this extra check + //upfront, because Data::LoadFromFile() doesn't crash if we give it the path of a directory + //instead the path of a file. + if(!bf::is_regular_file(filepath)) { + return nullptr; + } + Data data = Data::LoadFromFile(filepath); + return unique_ptr(new OnDiskBlock(filepath, std::move(data))); + } catch (const FileDoesntExistException &e) { + return nullptr; + } +} + +unique_ptr OnDiskBlock::CreateOnDisk(const bf::path &filepath, size_t size) { + if (bf::exists(filepath)) { + return nullptr; + } + + auto block = unique_ptr(new OnDiskBlock(filepath, size)); + block->_fillDataWithZeroes(); + block->_storeToDisk(); + return block; +} + +void OnDiskBlock::_fillDataWithZeroes() { + _data.FillWithZeroes(); +} + +void OnDiskBlock::_storeToDisk() const { + _data.StoreToFile(_filepath); +} + +void OnDiskBlock::flush() { + _storeToDisk(); +} + +} +} diff --git a/src/blockstore/implementations/ondisk/OnDiskBlock.h b/src/blockstore/implementations/ondisk/OnDiskBlock.h new file mode 100644 index 00000000..05584e12 --- /dev/null +++ b/src/blockstore/implementations/ondisk/OnDiskBlock.h @@ -0,0 +1,46 @@ +#pragma once +#ifndef BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_ +#define BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_ + +#include +#include +#include +#include + +#include "fspp/utils/macros.h" + +namespace blockstore { +namespace ondisk { +class OnDiskBlockStore; + +class OnDiskBlock: public Block { +public: + virtual ~OnDiskBlock(); + + static std::unique_ptr LoadFromDisk(const boost::filesystem::path &filepath); + static std::unique_ptr CreateOnDisk(const boost::filesystem::path &filepath, size_t size); + + void *data() override; + const void *data() const override; + + void flush() override; + + size_t size() const override; + +private: + const boost::filesystem::path _filepath; + Data _data; + + OnDiskBlock(const boost::filesystem::path &filepath, size_t size); + OnDiskBlock(const boost::filesystem::path &filepath, Data &&data); + + void _fillDataWithZeroes(); + void _storeToDisk() const; + + DISALLOW_COPY_AND_ASSIGN(OnDiskBlock); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp b/src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp new file mode 100644 index 00000000..447501a6 --- /dev/null +++ b/src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp @@ -0,0 +1,35 @@ +#include +#include +#include + +using std::unique_ptr; +using std::make_unique; +using std::string; +using std::mutex; +using std::lock_guard; + +namespace bf = boost::filesystem; + +namespace blockstore { +namespace ondisk { + +OnDiskBlockStore::OnDiskBlockStore(const boost::filesystem::path &rootdir) + : _rootdir(rootdir) {} + +unique_ptr OnDiskBlockStore::create(const std::string &key, size_t size) { + auto file_path = _rootdir / key; + auto block = OnDiskBlock::CreateOnDisk(file_path, size); + + if (!block) { + return nullptr; + } + return make_unique(key, std::move(block)); +} + +unique_ptr OnDiskBlockStore::load(const string &key) { + auto file_path = _rootdir / key; + return OnDiskBlock::LoadFromDisk(file_path); +} + +} +} diff --git a/src/blockstore/implementations/ondisk/OnDiskBlockStore.h b/src/blockstore/implementations/ondisk/OnDiskBlockStore.h new file mode 100644 index 00000000..2566ba95 --- /dev/null +++ b/src/blockstore/implementations/ondisk/OnDiskBlockStore.h @@ -0,0 +1,31 @@ +#pragma once +#ifndef BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_ +#define BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_ + +#include +#include + +#include "fspp/utils/macros.h" + +#include + +namespace blockstore { +namespace ondisk { + +class OnDiskBlockStore: public BlockStoreWithRandomKeys { +public: + OnDiskBlockStore(const boost::filesystem::path &rootdir); + + std::unique_ptr create(const std::string &key, size_t size) override; + std::unique_ptr load(const std::string &key) override; + +private: + const boost::filesystem::path _rootdir; + + DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore); +}; + +} +} + +#endif diff --git a/src/blockstore/interface/Block.h b/src/blockstore/interface/Block.h new file mode 100644 index 00000000..3a343e7e --- /dev/null +++ b/src/blockstore/interface/Block.h @@ -0,0 +1,24 @@ +#pragma once +#ifndef BLOCKSTORE_INTERFACE_BLOCK_H_ +#define BLOCKSTORE_INTERFACE_BLOCK_H_ + +#include + +namespace blockstore { + +class Block { +public: + virtual ~Block() {} + + virtual void *data() = 0; + virtual const void *data() const = 0; + + virtual void flush() = 0; + + virtual size_t size() const = 0; +}; + +} + + +#endif diff --git a/src/blockstore/interface/BlockStore.h b/src/blockstore/interface/BlockStore.h new file mode 100644 index 00000000..4b81caf4 --- /dev/null +++ b/src/blockstore/interface/BlockStore.h @@ -0,0 +1,29 @@ +#pragma once +#ifndef FSPP_BLOCKSTORE_BLOCKSTORE_H_ +#define FSPP_BLOCKSTORE_BLOCKSTORE_H_ + +#include +#include +#include +#include + + +namespace blockstore { + +//TODO Don't use string, but own class for keys? (better performance for all keys have same length) + +class BlockStore { +public: + virtual ~BlockStore() {} + + virtual BlockWithKey create(size_t size) = 0; + //TODO Use boost::optional (if key doesn't exist) + // Return nullptr if block with this key doesn't exists + virtual std::unique_ptr load(const std::string &key) = 0; + //TODO Needed for performance? Or is deleting loaded blocks enough? + //virtual void remove(const std::string &key) = 0; +}; + +} + +#endif diff --git a/src/blockstore/interface/CMakeLists.txt b/src/blockstore/interface/CMakeLists.txt new file mode 100644 index 00000000..98e1c3cb --- /dev/null +++ b/src/blockstore/interface/CMakeLists.txt @@ -0,0 +1 @@ +add_library(blockstore_interface helpers/BlockStoreWithRandomKeys.cpp) diff --git a/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.cpp b/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.cpp new file mode 100644 index 00000000..f654746b --- /dev/null +++ b/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.cpp @@ -0,0 +1,19 @@ +#include +#include + +using namespace blockstore; + +using std::string; + +BlockWithKey BlockStoreWithRandomKeys::create(size_t size) { + std::unique_ptr result; + do { + result = create(_generateRandomKey(), size); + } while (!result); + + return std::move(*result); +} + +string BlockStoreWithRandomKeys::_generateRandomKey() { + return RandomKeyGenerator::singleton().create(); +} diff --git a/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.h b/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.h new file mode 100644 index 00000000..765be79b --- /dev/null +++ b/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.h @@ -0,0 +1,27 @@ +#pragma once +#ifndef FSPP_BLOCKSTORE_BLOCKSTOREWITHRANDOMKEYS_H_ +#define FSPP_BLOCKSTORE_BLOCKSTOREWITHRANDOMKEYS_H_ + +#include +#include + +namespace blockstore { + +// This is an implementation helpers for BlockStores that use random block keys. +// You should never give this static type to the client. The client should always +// work with the BlockStore interface instead. +class BlockStoreWithRandomKeys: public BlockStore { +public: + //TODO Use boost::optional (if key already exists) + // Return nullptr if key already exists + virtual std::unique_ptr create(const std::string &key, size_t size) = 0; + + BlockWithKey create(size_t size) final; + +private: + std::string _generateRandomKey(); +}; + +} + +#endif diff --git a/src/blockstore/utils/BlockWithKey.h b/src/blockstore/utils/BlockWithKey.h new file mode 100644 index 00000000..15135640 --- /dev/null +++ b/src/blockstore/utils/BlockWithKey.h @@ -0,0 +1,20 @@ +#pragma once +#ifndef BLOCKSTORE_INTERFACE_BLOCKWITHKEY_H_ +#define BLOCKSTORE_INTERFACE_BLOCKWITHKEY_H_ + +#include +#include +#include "fspp/utils/macros.h" + +namespace blockstore { + +struct BlockWithKey { + BlockWithKey(const std::string &key_, std::unique_ptr block_): key(key_), block(std::move(block_)) {} + + std::string key; + std::unique_ptr block; +}; + +} + +#endif diff --git a/src/blockstore/utils/CMakeLists.txt b/src/blockstore/utils/CMakeLists.txt new file mode 100644 index 00000000..d8dfc93e --- /dev/null +++ b/src/blockstore/utils/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(blockstore_utils Data.cpp RandomKeyGenerator.cpp FileDoesntExistException.cpp) + +target_link_libraries(blockstore_utils cryptopp) diff --git a/src/blockstore/utils/Data.cpp b/src/blockstore/utils/Data.cpp new file mode 100644 index 00000000..db4ec5df --- /dev/null +++ b/src/blockstore/utils/Data.cpp @@ -0,0 +1,91 @@ +#include +#include +#include "FileDoesntExistException.h" + +#include +#include + +using std::istream; +using std::ofstream; +using std::ifstream; +using std::ios; +using std::unique_ptr; +using std::make_unique; + +namespace bf = boost::filesystem; + +namespace blockstore { + +Data::Data(size_t size) +: _size(size), _data(std::malloc(size)) { + if (nullptr == _data) { + throw std::bad_alloc(); + } +} + +Data::Data(Data &&rhs) +: _size(rhs._size), _data(rhs._data) { + // Make rhs invalid, so the memory doesn't get freed in its destructor. + rhs._data = nullptr; +} + +Data::~Data() { + std::free(_data); + _data = nullptr; +} + +void *Data::data() { + return const_cast(const_cast(this)->data()); +} + +const void *Data::data() const { + return _data; +} + +size_t Data::size() const { + return _size; +} + +void Data::FillWithZeroes() { + std::memset(_data, 0, _size); +} + +void Data::StoreToFile(const bf::path &filepath) const { + ofstream file(filepath.c_str(), ios::binary | ios::trunc); + file.write((const char*)_data, _size); +} + +Data Data::LoadFromFile(const bf::path &filepath) { + ifstream file(filepath.c_str(), ios::binary); + _assertFileExists(file, filepath); + size_t size = _getStreamSize(file); + + Data result(size); + result._readFromStream(file); + return result; +} + +void Data::_assertFileExists(const ifstream &file, const bf::path &filepath) { + if (!file.good()) { + throw FileDoesntExistException(filepath); + } +} + +size_t Data::_getStreamSize(istream &stream) { + auto current_pos = stream.tellg(); + + //Retrieve length + stream.seekg(0, stream.end); + auto endpos = stream.tellg(); + + //Restore old position + stream.seekg(current_pos, stream.beg); + + return endpos - current_pos; +} + +void Data::_readFromStream(istream &stream) { + stream.read((char*)_data, _size); +} + +} diff --git a/src/blockstore/utils/Data.h b/src/blockstore/utils/Data.h new file mode 100644 index 00000000..606aeada --- /dev/null +++ b/src/blockstore/utils/Data.h @@ -0,0 +1,43 @@ +#pragma once +#ifndef BLOCKSTORE_IMPLEMENTATIONS_ONDISK_DATA_H_ +#define BLOCKSTORE_IMPLEMENTATIONS_ONDISK_DATA_H_ + +#include +//TODO Move this to a more generic utils +#include "fspp/utils/macros.h" + +#include +#include + +namespace blockstore { + +class Data { +public: + Data(size_t size); + Data(Data &&rhs); // move constructor + virtual ~Data(); + + void *data(); + const void *data() const; + + size_t size() const; + + void FillWithZeroes(); + + void StoreToFile(const boost::filesystem::path &filepath) const; + static Data LoadFromFile(const boost::filesystem::path &filepath); + +private: + size_t _size; + void *_data; + + static void _assertFileExists(const std::ifstream &file, const boost::filesystem::path &filepath); + static size_t _getStreamSize(std::istream &stream); + void _readFromStream(std::istream &stream); + + DISALLOW_COPY_AND_ASSIGN(Data); +}; + +} + +#endif diff --git a/src/blockstore/utils/FileDoesntExistException.cpp b/src/blockstore/utils/FileDoesntExistException.cpp new file mode 100644 index 00000000..c65dd22a --- /dev/null +++ b/src/blockstore/utils/FileDoesntExistException.cpp @@ -0,0 +1,17 @@ +#include + +namespace bf = boost::filesystem; + +using std::runtime_error; +using std::string; + +namespace blockstore { + +FileDoesntExistException::FileDoesntExistException(const bf::path &filepath) +: runtime_error(string("The file ")+filepath.c_str()+" doesn't exist") { +} + +FileDoesntExistException::~FileDoesntExistException() { +} + +} diff --git a/src/blockstore/utils/FileDoesntExistException.h b/src/blockstore/utils/FileDoesntExistException.h new file mode 100644 index 00000000..d4c0a521 --- /dev/null +++ b/src/blockstore/utils/FileDoesntExistException.h @@ -0,0 +1,19 @@ +#pragma once +#ifndef BLOCKSTORE_UTILS_FILEDOESNTEXISTEXCEPTION_H_ +#define BLOCKSTORE_UTILS_FILEDOESNTEXISTEXCEPTION_H_ + +#include + +#include + +namespace blockstore { + +class FileDoesntExistException: public std::runtime_error { +public: + FileDoesntExistException(const boost::filesystem::path &filepath); + virtual ~FileDoesntExistException(); +}; + +} + +#endif diff --git a/src/blockstore/utils/RandomKeyGenerator.cpp b/src/blockstore/utils/RandomKeyGenerator.cpp new file mode 100644 index 00000000..4f294fc3 --- /dev/null +++ b/src/blockstore/utils/RandomKeyGenerator.cpp @@ -0,0 +1,53 @@ +#include + +using std::string; + +#include +#include + +using CryptoPP::AutoSeededRandomPool; +using CryptoPP::ArraySource; +using CryptoPP::StringSink; +using CryptoPP::HexEncoder; + +using std::make_unique; + +namespace blockstore { + +constexpr unsigned int RandomKeyGenerator::KEYLENGTH_ENTROPY; +constexpr unsigned int RandomKeyGenerator::KEYLENGTH; + +namespace { +string encodeKeyToHex(const byte *data); +} + +RandomKeyGenerator::RandomKeyGenerator() +: _randomPool(make_unique()) { +} + +RandomKeyGenerator::~RandomKeyGenerator() { +} + +RandomKeyGenerator &RandomKeyGenerator::singleton() { + static RandomKeyGenerator singleton; + return singleton; +} + +string RandomKeyGenerator::create() { + byte key[KEYLENGTH_ENTROPY]; + _randomPool->GenerateBlock(key, KEYLENGTH_ENTROPY); + return encodeKeyToHex(key); +} + +namespace { +string encodeKeyToHex(const byte *data) { + string result; + ArraySource(data, RandomKeyGenerator::KEYLENGTH_ENTROPY, true, + new HexEncoder(new StringSink(result)) + ); + assert(result.size() == RandomKeyGenerator::KEYLENGTH); + return result; +} +} + +} diff --git a/src/blockstore/utils/RandomKeyGenerator.h b/src/blockstore/utils/RandomKeyGenerator.h new file mode 100644 index 00000000..275c6b72 --- /dev/null +++ b/src/blockstore/utils/RandomKeyGenerator.h @@ -0,0 +1,37 @@ +#pragma once +#ifndef BLOCKSTORE_UTILS_RANDOMKEYGENERATOR_H_ +#define BLOCKSTORE_UTILS_RANDOMKEYGENERATOR_H_ + +#include "fspp/utils/macros.h" +#include + +namespace CryptoPP { +class AutoSeededRandomPool; +} + +namespace blockstore { + +// Creates random keys for use as block access handles. +// A key here is NOT a key for encryption, but a key as used in key->value mappings ("access handle for a block"). +class RandomKeyGenerator { +public: + virtual ~RandomKeyGenerator(); + + static constexpr unsigned int KEYLENGTH_ENTROPY = 16; // random bytes in the key + static constexpr unsigned int KEYLENGTH = KEYLENGTH_ENTROPY * 2; + + static RandomKeyGenerator &singleton(); + + std::string create(); + +private: + RandomKeyGenerator(); + + std::unique_ptr _randomPool; + + DISALLOW_COPY_AND_ASSIGN(RandomKeyGenerator); +}; + +} + +#endif diff --git a/src/test/blockstore/implementations/inmemory/InMemoryBlockStoreTest.cpp b/src/test/blockstore/implementations/inmemory/InMemoryBlockStoreTest.cpp new file mode 100644 index 00000000..b6d37261 --- /dev/null +++ b/src/test/blockstore/implementations/inmemory/InMemoryBlockStoreTest.cpp @@ -0,0 +1,31 @@ +#include +#include +#include +#include +#include "gtest/gtest.h" + + +using blockstore::BlockStore; +using blockstore::BlockStoreWithRandomKeys; +using blockstore::inmemory::InMemoryBlockStore; + +using std::unique_ptr; +using std::make_unique; + +class InMemoryBlockStoreTestFixture: public BlockStoreTestFixture { +public: + unique_ptr createBlockStore() override { + return make_unique(); + } +}; + +INSTANTIATE_TYPED_TEST_CASE_P(InMemory, BlockStoreTest, InMemoryBlockStoreTestFixture); + +class InMemoryBlockStoreWithRandomKeysTestFixture: public BlockStoreWithRandomKeysTestFixture { +public: + unique_ptr createBlockStore() override { + return make_unique(); + } +}; + +INSTANTIATE_TYPED_TEST_CASE_P(InMemory, BlockStoreWithRandomKeysTest, InMemoryBlockStoreWithRandomKeysTestFixture); diff --git a/src/test/blockstore/implementations/ondisk/OnDiskBlockStoreTest.cpp b/src/test/blockstore/implementations/ondisk/OnDiskBlockStoreTest.cpp new file mode 100644 index 00000000..a6ee4195 --- /dev/null +++ b/src/test/blockstore/implementations/ondisk/OnDiskBlockStoreTest.cpp @@ -0,0 +1,35 @@ +#include +#include +#include +#include +#include "gtest/gtest.h" + + +using blockstore::BlockStore; +using blockstore::BlockStoreWithRandomKeys; +using blockstore::ondisk::OnDiskBlockStore; + +using std::unique_ptr; +using std::make_unique; + +class OnDiskBlockStoreTestFixture: public BlockStoreTestFixture { +public: + unique_ptr createBlockStore() override { + return make_unique(tempdir.path()); + } +private: + TempDir tempdir; +}; + +INSTANTIATE_TYPED_TEST_CASE_P(OnDisk, BlockStoreTest, OnDiskBlockStoreTestFixture); + +class OnDiskBlockStoreWithRandomKeysTestFixture: public BlockStoreWithRandomKeysTestFixture { +public: + unique_ptr createBlockStore() override { + return make_unique(tempdir.path()); + } +private: + TempDir tempdir; +}; + +INSTANTIATE_TYPED_TEST_CASE_P(OnDisk, BlockStoreWithRandomKeysTest, OnDiskBlockStoreWithRandomKeysTestFixture); diff --git a/src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp b/src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp new file mode 100644 index 00000000..9c1fa9ce --- /dev/null +++ b/src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp @@ -0,0 +1,78 @@ +#include +#include +#include "gtest/gtest.h" + +#include "test/testutils/TempFile.h" + + +using ::testing::Test; +using ::testing::WithParamInterface; +using ::testing::Values; + +using std::unique_ptr; + +using namespace blockstore; +using namespace blockstore::ondisk; + +namespace bf = boost::filesystem; + +class OnDiskBlockCreateTest: public Test { +public: + OnDiskBlockCreateTest() + // Don't create the temp file yet (therefore pass false to the TempFile constructor) + : file(false) { + } + TempFile file; +}; + +TEST_F(OnDiskBlockCreateTest, CreatingBlockCreatesFile) { + EXPECT_FALSE(bf::exists(file.path())); + + auto block = OnDiskBlock::CreateOnDisk(file.path(), 0); + + EXPECT_TRUE(bf::exists(file.path())); + EXPECT_TRUE(bf::is_regular_file(file.path())); +} + +TEST_F(OnDiskBlockCreateTest, CreatingExistingBlockReturnsNull) { + auto block1 = OnDiskBlock::CreateOnDisk(file.path(), 0); + auto block2 = OnDiskBlock::CreateOnDisk(file.path(), 0); + EXPECT_TRUE((bool)block1); + EXPECT_FALSE((bool)block2); +} + +class OnDiskBlockCreateSizeTest: public OnDiskBlockCreateTest, public WithParamInterface { +public: + unique_ptr block; + Data ZEROES; + + OnDiskBlockCreateSizeTest(): + block(OnDiskBlock::CreateOnDisk(file.path(), GetParam())), + ZEROES(block->size()) + { + ZEROES.FillWithZeroes(); + } +}; +INSTANTIATE_TEST_CASE_P(OnDiskBlockCreateSizeTest, OnDiskBlockCreateSizeTest, Values(0, 1, 5, 1024, 10*1024*1024)); + +TEST_P(OnDiskBlockCreateSizeTest, OnDiskSizeIsCorrect) { + Data fileContent = Data::LoadFromFile(file.path()); + EXPECT_EQ(GetParam(), fileContent.size()); +} + +TEST_P(OnDiskBlockCreateSizeTest, OnDiskBlockIsZeroedOut) { + Data fileContent = Data::LoadFromFile(file.path()); + EXPECT_EQ(0, std::memcmp(ZEROES.data(), fileContent.data(), fileContent.size())); +} + +// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface. +// Here, we create it using OnDiskBlock::CreateOnDisk() +TEST_P(OnDiskBlockCreateSizeTest, InMemorySizeIsCorrect) { + EXPECT_EQ(GetParam(), block->size()); +} + +// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface. +// Here, we create it using OnDiskBlock::CreateOnDisk() +TEST_P(OnDiskBlockCreateSizeTest, InMemoryBlockIsZeroedOut) { + EXPECT_EQ(0, std::memcmp(ZEROES.data(), block->data(), block->size())); +} diff --git a/src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp b/src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp new file mode 100644 index 00000000..eb279ca7 --- /dev/null +++ b/src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp @@ -0,0 +1,111 @@ +#include +#include +#include +#include "gtest/gtest.h" + +#include "test/testutils/TempFile.h" + +using ::testing::Test; +using ::testing::WithParamInterface; +using ::testing::Values; + +using std::unique_ptr; + +using namespace blockstore; +using namespace blockstore::ondisk; + +namespace bf = boost::filesystem; + +class OnDiskBlockFlushTest: public Test, public WithParamInterface { +public: + OnDiskBlockFlushTest() + // Don't create the temp file yet (therefore pass false to the TempFile constructor) + : file(false), randomData(GetParam()) { + } + TempFile file; + + DataBlockFixture randomData; + + unique_ptr CreateBlockAndLoadItFromDisk() { + { + auto block = OnDiskBlock::CreateOnDisk(file.path(), randomData.size()); + } + return OnDiskBlock::LoadFromDisk(file.path()); + } + + unique_ptr CreateBlock() { + return OnDiskBlock::CreateOnDisk(file.path(), randomData.size()); + } + + void WriteDataToBlock(const unique_ptr &block) { + std::memcpy(block->data(), randomData.data(), randomData.size()); + } + + void EXPECT_BLOCK_DATA_CORRECT(const unique_ptr &block) { + EXPECT_EQ(randomData.size(), block->size()); + EXPECT_EQ(0, std::memcmp(randomData.data(), block->data(), randomData.size())); + } + + void EXPECT_STORED_FILE_DATA_CORRECT() { + Data actual = Data::LoadFromFile(file.path()); + EXPECT_EQ(randomData.size(), actual.size()); + EXPECT_EQ(0, std::memcmp(randomData.data(), actual.data(), randomData.size())); + } +}; +INSTANTIATE_TEST_CASE_P(OnDiskBlockFlushTest, OnDiskBlockFlushTest, Values((size_t)0, (size_t)1, (size_t)1024, (size_t)4096, (size_t)10*1024*1024)); + +// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface. +// Here, we create it using OnDiskBlock::CreateOnDisk() +TEST_P(OnDiskBlockFlushTest, AfterCreate_FlushingDoesntChangeBlock) { + auto block = CreateBlock(); + WriteDataToBlock(block); + block->flush(); + + EXPECT_BLOCK_DATA_CORRECT(block); +} + +// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface. +// Here, we create it using OnDiskBlock::CreateOnDisk() / OnDiskBlock::LoadFromDisk() +TEST_P(OnDiskBlockFlushTest, AfterLoad_FlushingDoesntChangeBlock) { + auto block = CreateBlockAndLoadItFromDisk(); + WriteDataToBlock(block); + block->flush(); + + EXPECT_BLOCK_DATA_CORRECT(block); +} + +TEST_P(OnDiskBlockFlushTest, AfterCreate_FlushingWritesCorrectData) { + auto block = CreateBlock(); + WriteDataToBlock(block); + block->flush(); + + EXPECT_STORED_FILE_DATA_CORRECT(); +} + +TEST_P(OnDiskBlockFlushTest, AfterLoad_FlushingWritesCorrectData) { + auto block = CreateBlockAndLoadItFromDisk(); + WriteDataToBlock(block); + block->flush(); + + EXPECT_STORED_FILE_DATA_CORRECT(); +} + +// This test is also tested by OnDiskBlockStoreTest, but there it can only checks block content by loading it again. +// Here, we check the content on disk. +TEST_P(OnDiskBlockFlushTest, AfterCreate_FlushesWhenDestructed) { + { + auto block = CreateBlock(); + WriteDataToBlock(block); + } + EXPECT_STORED_FILE_DATA_CORRECT(); +} + +// This test is also tested by OnDiskBlockStoreTest, but there it can only checks block content by loading it again. +// Here, we check the content on disk. +TEST_P(OnDiskBlockFlushTest, AfterLoad_FlushesWhenDestructed) { + { + auto block = CreateBlockAndLoadItFromDisk(); + WriteDataToBlock(block); + } + EXPECT_STORED_FILE_DATA_CORRECT(); +} diff --git a/src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp b/src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp new file mode 100644 index 00000000..f97013b3 --- /dev/null +++ b/src/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include +#include "gtest/gtest.h" + +#include "test/testutils/TempFile.h" +#include + +using ::testing::Test; +using ::testing::WithParamInterface; +using ::testing::Values; + +using std::ofstream; +using std::unique_ptr; +using std::ios; + +using namespace blockstore; +using namespace blockstore::ondisk; + +namespace bf = boost::filesystem; + +class OnDiskBlockLoadTest: public Test, public WithParamInterface { +public: + TempFile file; + + void SetFileSize(size_t size) { + Data data(size); + data.StoreToFile(file.path()); + } + + void StoreData(const DataBlockFixture &data) { + //TODO Implement data.StoreToFile(filepath) instead + Data dataobj(data.size()); + std::memcpy(dataobj.data(), data.data(), data.size()); + dataobj.StoreToFile(file.path()); + } + + unique_ptr LoadBlock() { + return OnDiskBlock::LoadFromDisk(file.path()); + } + + void EXPECT_BLOCK_DATA_EQ(const DataBlockFixture &expected, const OnDiskBlock &actual) { + EXPECT_EQ(expected.size(), actual.size()); + EXPECT_EQ(0, std::memcmp(expected.data(), actual.data(), expected.size())); + } +}; +INSTANTIATE_TEST_CASE_P(OnDiskBlockLoadTest, OnDiskBlockLoadTest, Values(0, 1, 5, 1024, 10*1024*1024)); + +TEST_P(OnDiskBlockLoadTest, FileSizeIsCorrect) { + SetFileSize(GetParam()); + + auto block = LoadBlock(); + + EXPECT_EQ(GetParam(), block->size()); +} + +TEST_P(OnDiskBlockLoadTest, LoadedDataIsCorrect) { + DataBlockFixture randomData(GetParam()); + StoreData(randomData); + + auto block = LoadBlock(); + + EXPECT_BLOCK_DATA_EQ(randomData, *block); +} + +TEST_F(OnDiskBlockLoadTest, LoadNotExistingBlock) { + TempFile file2(false); // Pass false, so the file isn't created. + EXPECT_FALSE( + (bool)OnDiskBlock::LoadFromDisk(file2.path()) + ); +} diff --git a/src/test/blockstore/interface/BlockStoreTest.cpp b/src/test/blockstore/interface/BlockStoreTest.cpp new file mode 100644 index 00000000..23a03496 --- /dev/null +++ b/src/test/blockstore/interface/BlockStoreTest.cpp @@ -0,0 +1,4 @@ +/* + * Tests that the header can be included without needing additional header includes as dependencies. + */ +#include diff --git a/src/test/blockstore/interface/BlockTest.cpp b/src/test/blockstore/interface/BlockTest.cpp new file mode 100644 index 00000000..bf85aad0 --- /dev/null +++ b/src/test/blockstore/interface/BlockTest.cpp @@ -0,0 +1,4 @@ +/* + * Tests that the header can be included without needing additional header includes as dependencies. + */ +#include diff --git a/src/test/blockstore/interface/helpers/BlockStoreWithRandomKeysTest.cpp b/src/test/blockstore/interface/helpers/BlockStoreWithRandomKeysTest.cpp new file mode 100644 index 00000000..d0e2af98 --- /dev/null +++ b/src/test/blockstore/interface/helpers/BlockStoreWithRandomKeysTest.cpp @@ -0,0 +1,117 @@ +#include +#include +#include +#include + + +using ::testing::Test; +using ::testing::_; +using ::testing::Return; +using ::testing::Invoke; + +using std::string; +using std::unique_ptr; +using std::make_unique; + +using namespace blockstore; + +class BlockStoreWithRandomKeysMock: public BlockStoreWithRandomKeys { +public: + unique_ptr create(const std::string &key, size_t size) { + return unique_ptr(do_create(key, size)); + } + MOCK_METHOD2(do_create, BlockWithKey*(const std::string &, size_t)); + unique_ptr load(const string &key) { + return unique_ptr(do_load(key)); + } + MOCK_METHOD1(do_load, Block*(const string &)); + MOCK_METHOD1(exists, bool(const string &)); +}; + +class BlockMock: public Block { +public: + MOCK_METHOD0(data, void*()); + MOCK_CONST_METHOD0(data, const void*()); + MOCK_METHOD0(flush, void()); + MOCK_CONST_METHOD0(size, size_t()); +}; + +class BlockStoreWithRandomKeysTest: public Test { +public: + BlockStoreWithRandomKeysMock blockStoreMock; + BlockStore &blockStore = blockStoreMock; +}; + +TEST_F(BlockStoreWithRandomKeysTest, SizeIsPassedThrough0) { + EXPECT_CALL(blockStoreMock, do_create(_, 0)).WillOnce(Return(new BlockWithKey("", make_unique()))); + blockStore.create(0); +} + +TEST_F(BlockStoreWithRandomKeysTest, SizeIsPassedThrough1) { + EXPECT_CALL(blockStoreMock, do_create(_, 1)).WillOnce(Return(new BlockWithKey("", make_unique()))); + blockStore.create(1); +} + +TEST_F(BlockStoreWithRandomKeysTest, SizeIsPassedThrough1024) { + EXPECT_CALL(blockStoreMock, do_create(_, 1024)).WillOnce(Return(new BlockWithKey("", make_unique()))); + blockStore.create(1024); +} + +TEST_F(BlockStoreWithRandomKeysTest, KeyHasCorrectSize) { + EXPECT_CALL(blockStoreMock, do_create(_, _)).WillOnce(Invoke([](const string &key, size_t) { + EXPECT_EQ(RandomKeyGenerator::KEYLENGTH, key.size()); + return new BlockWithKey("", make_unique()); + })); + + blockStore.create(1024); +} + +TEST_F(BlockStoreWithRandomKeysTest, TwoBlocksGetDifferentKeys) { + string first_key; + EXPECT_CALL(blockStoreMock, do_create(_, _)) + .WillOnce(Invoke([&first_key](const string &key, size_t) { + first_key = key; + return new BlockWithKey("", make_unique()); + })) + .WillOnce(Invoke([&first_key](const string &key, size_t) { + EXPECT_NE(first_key, key); + return new BlockWithKey("", make_unique()); + })); + + blockStore.create(1024); + blockStore.create(1024); +} + +TEST_F(BlockStoreWithRandomKeysTest, WillTryADifferentKeyIfKeyAlreadyExists) { + string first_key; + EXPECT_CALL(blockStoreMock, do_create(_, _)) + .WillOnce(Invoke([&first_key](const string &key, size_t) { + first_key = key; + return nullptr; + })) + .WillOnce(Invoke([&first_key](const string &key, size_t) { + EXPECT_NE(first_key, key); + return new BlockWithKey("", make_unique()); + })); + + blockStore.create(1024); +} + +TEST_F(BlockStoreWithRandomKeysTest, WillTryADifferentKeyIfKeyAlreadyExistsTwoTimes) { + string first_key; + EXPECT_CALL(blockStoreMock, do_create(_, _)) + .WillOnce(Invoke([&first_key](const string &key, size_t) { + first_key = key; + return nullptr; + })) + .WillOnce(Invoke([&first_key](const string &key, size_t) { + first_key = key; + return nullptr; + })) + .WillOnce(Invoke([&first_key](const string &key, size_t) { + EXPECT_NE(first_key, key); + return new BlockWithKey("", make_unique()); + })); + + blockStore.create(1024); +} diff --git a/src/test/blockstore/testutils/BlockStoreTest.h b/src/test/blockstore/testutils/BlockStoreTest.h new file mode 100644 index 00000000..32ccf289 --- /dev/null +++ b/src/test/blockstore/testutils/BlockStoreTest.h @@ -0,0 +1,226 @@ +#pragma once +#ifndef TEST_BLOCKSTORE_IMPLEMENTATIONS_TESTUTILS_BLOCKSTORETEST_H_ +#define TEST_BLOCKSTORE_IMPLEMENTATIONS_TESTUTILS_BLOCKSTORETEST_H_ + +#include +#include +#include +#include "test/testutils/TempDir.h" + +class BlockStoreTestFixture { +public: + virtual std::unique_ptr createBlockStore() = 0; +}; + +template +class BlockStoreTest: public ::testing::Test { +public: + BOOST_STATIC_ASSERT_MSG( + (std::is_base_of::value), + "Given test fixture for instantiating the (type parameterized) BlockStoreTest must inherit from BlockStoreTestFixture" + ); + + const std::vector SIZES = {0, 1, 1024, 4096, 10*1024*1024}; + + ConcreteBlockStoreTestFixture fixture; +}; + +template +class BlockStoreSizeParameterizedTest { +public: + BlockStoreSizeParameterizedTest(ConcreateBlockStoreTestFixture &fixture, size_t size_): blockStore(fixture.createBlockStore()), size(size_) {} + + void TestCreatedBlockHasCorrectSize() { + auto block = blockStore->create(size); + EXPECT_EQ(size, block.block->size()); + } + + void TestLoadingUnchangedBlockHasCorrectSize() { + auto block = blockStore->create(size); + auto loaded_block = blockStore->load(block.key); + EXPECT_EQ(size, loaded_block->size()); + } + + void TestCreatedBlockIsZeroedOut() { + auto block = blockStore->create(size); + EXPECT_EQ(0, std::memcmp(ZEROES(size).data(), block.block->data(), size)); + } + + void TestLoadingUnchangedBlockIsZeroedOut() { + auto block = blockStore->create(size); + auto loaded_block = blockStore->load(block.key); + EXPECT_EQ(0, std::memcmp(ZEROES(size).data(), loaded_block->data(), size)); + } + + void TestLoadedBlockIsCorrect() { + DataBlockFixture randomData(size); + auto loaded_block = StoreDataToBlockAndLoadIt(randomData); + EXPECT_EQ(size, loaded_block->size()); + EXPECT_EQ(0, std::memcmp(randomData.data(), loaded_block->data(), size)); + } + + void TestLoadedBlockIsCorrectWhenLoadedDirectlyAfterFlushing() { + DataBlockFixture randomData(size); + auto loaded_block = StoreDataToBlockAndLoadItDirectlyAfterFlushing(randomData); + EXPECT_EQ(size, loaded_block->size()); + EXPECT_EQ(0, std::memcmp(randomData.data(), loaded_block->data(), size)); + } + + void TestAfterCreate_FlushingDoesntChangeBlock() { + DataBlockFixture randomData(size); + auto block = CreateBlock(); + WriteDataToBlock(block.get(), randomData); + block->flush(); + + EXPECT_BLOCK_DATA_CORRECT(*block, randomData); + } + + void TestAfterLoad_FlushingDoesntChangeBlock() { + DataBlockFixture randomData(size); + auto block = CreateBlockAndLoadIt(); + WriteDataToBlock(block.get(), randomData); + block->flush(); + + EXPECT_BLOCK_DATA_CORRECT(*block, randomData); + } + + void TestAfterCreate_FlushesWhenDestructed() { + DataBlockFixture randomData(size); + std::string key; + { + auto block = blockStore->create(size); + key = block.key; + WriteDataToBlock(block.block.get(), randomData); + } + auto loaded_block = blockStore->load(key); + EXPECT_BLOCK_DATA_CORRECT(*loaded_block, randomData); + } + + void TestAfterLoad_FlushesWhenDestructed() { + DataBlockFixture randomData(size); + std::string key; + { + key = blockStore->create(size).key; + auto block = blockStore->load(key); + WriteDataToBlock(block.get(), randomData); + } + auto loaded_block = blockStore->load(key); + EXPECT_BLOCK_DATA_CORRECT(*loaded_block, randomData); + } + + void TestLoadNonExistingBlockWithDefinitelyValidKey() { + EXPECT_FALSE( + (bool)blockStore->load(blockstore::RandomKeyGenerator::singleton().create()) + ); + } + + void TestLoadNonExistingBlockWithMaybeInvalidKey() { + EXPECT_FALSE( + (bool)blockStore->load("not-existing-key") + ); + } + + void TestLoadNonExistingBlockWithEmptyKey() { + EXPECT_FALSE( + (bool)blockStore->load("") + ); + } + +private: + std::unique_ptr blockStore; + size_t size; + + blockstore::Data ZEROES(size_t size) { + blockstore::Data ZEROES(size); + ZEROES.FillWithZeroes(); + return ZEROES; + } + + std::unique_ptr StoreDataToBlockAndLoadIt(const DataBlockFixture &data) { + std::string key = StoreDataToBlockAndGetKey(data); + return blockStore->load(key); + } + + std::string StoreDataToBlockAndGetKey(const DataBlockFixture &data) { + auto block = blockStore->create(data.size()); + std::memcpy(block.block->data(), data.data(), data.size()); + return block.key; + } + + std::unique_ptr StoreDataToBlockAndLoadItDirectlyAfterFlushing(const DataBlockFixture &data) { + auto block = blockStore->create(data.size()); + std::memcpy(block.block->data(), data.data(), data.size()); + block.block->flush(); + return blockStore->load(block.key); + } + + std::unique_ptr CreateBlockAndLoadIt() { + std::string key = blockStore->create(size).key; + return blockStore->load(key); + } + + std::unique_ptr CreateBlock() { + return blockStore->create(size).block; + } + + void WriteDataToBlock(blockstore::Block *block, const DataBlockFixture &randomData) { + std::memcpy(block->data(), randomData.data(), randomData.size()); + } + + void EXPECT_BLOCK_DATA_CORRECT(const blockstore::Block &block, const DataBlockFixture &randomData) { + EXPECT_EQ(randomData.size(), block.size()); + EXPECT_EQ(0, std::memcmp(randomData.data(), block.data(), randomData.size())); + } +}; + +TYPED_TEST_CASE_P(BlockStoreTest); + +#define TYPED_TEST_P_FOR_ALL_SIZES(TestName) \ + TYPED_TEST_P(BlockStoreTest, TestName) { \ + for (auto size: this->SIZES) { \ + BlockStoreSizeParameterizedTest(this->fixture, size) \ + .Test##TestName(); \ + } \ + } \ + + +TYPED_TEST_P_FOR_ALL_SIZES(CreatedBlockHasCorrectSize); +TYPED_TEST_P_FOR_ALL_SIZES(LoadingUnchangedBlockHasCorrectSize); +TYPED_TEST_P_FOR_ALL_SIZES(CreatedBlockIsZeroedOut); +TYPED_TEST_P_FOR_ALL_SIZES(LoadingUnchangedBlockIsZeroedOut); +TYPED_TEST_P_FOR_ALL_SIZES(LoadedBlockIsCorrect); +TYPED_TEST_P_FOR_ALL_SIZES(LoadedBlockIsCorrectWhenLoadedDirectlyAfterFlushing); +TYPED_TEST_P_FOR_ALL_SIZES(AfterCreate_FlushingDoesntChangeBlock); +TYPED_TEST_P_FOR_ALL_SIZES(AfterLoad_FlushingDoesntChangeBlock); +TYPED_TEST_P_FOR_ALL_SIZES(AfterCreate_FlushesWhenDestructed); +TYPED_TEST_P_FOR_ALL_SIZES(AfterLoad_FlushesWhenDestructed); +TYPED_TEST_P_FOR_ALL_SIZES(LoadNonExistingBlockWithDefinitelyValidKey); +TYPED_TEST_P_FOR_ALL_SIZES(LoadNonExistingBlockWithMaybeInvalidKey); +TYPED_TEST_P_FOR_ALL_SIZES(LoadNonExistingBlockWithEmptyKey); + +TYPED_TEST_P(BlockStoreTest, TwoCreatedBlocksHaveDifferentKeys) { + auto blockStore = this->fixture.createBlockStore(); + auto block1 = blockStore->create(1024); + auto block2 = blockStore->create(1024); + EXPECT_NE(block1.key, block2.key); +} + +REGISTER_TYPED_TEST_CASE_P(BlockStoreTest, + CreatedBlockHasCorrectSize, + LoadingUnchangedBlockHasCorrectSize, + CreatedBlockIsZeroedOut, + LoadingUnchangedBlockIsZeroedOut, + LoadedBlockIsCorrect, + LoadedBlockIsCorrectWhenLoadedDirectlyAfterFlushing, + AfterCreate_FlushingDoesntChangeBlock, + AfterLoad_FlushingDoesntChangeBlock, + AfterCreate_FlushesWhenDestructed, + AfterLoad_FlushesWhenDestructed, + LoadNonExistingBlockWithDefinitelyValidKey, + LoadNonExistingBlockWithMaybeInvalidKey, + LoadNonExistingBlockWithEmptyKey, + TwoCreatedBlocksHaveDifferentKeys +); + + +#endif diff --git a/src/test/blockstore/testutils/BlockStoreWithRandomKeysTest.h b/src/test/blockstore/testutils/BlockStoreWithRandomKeysTest.h new file mode 100644 index 00000000..d379f8c9 --- /dev/null +++ b/src/test/blockstore/testutils/BlockStoreWithRandomKeysTest.h @@ -0,0 +1,81 @@ +#pragma once +#ifndef TEST_BLOCKSTORE_IMPLEMENTATIONS_TESTUTILS_BLOCKSTOREWITHRANDOMKEYSTEST_H_ +#define TEST_BLOCKSTORE_IMPLEMENTATIONS_TESTUTILS_BLOCKSTOREWITHRANDOMKEYSTEST_H_ + +#include + +#include +#include +#include "test/testutils/TempDir.h" +#include "blockstore/utils/RandomKeyGenerator.h" + +class BlockStoreWithRandomKeysTestFixture { +public: + virtual std::unique_ptr createBlockStore() = 0; +}; + +template +class BlockStoreWithRandomKeysTest: public ::testing::Test { +public: + BOOST_STATIC_ASSERT_MSG( + (std::is_base_of::value), + "Given test fixture for instantiating the (type parameterized) BlockStoreWithRandomKeysTest must inherit from BlockStoreWithRandomKeysTestFixture" + ); + + const std::vector SIZES = {0, 1, 1024, 4096, 10*1024*1024}; + + ConcreteBlockStoreWithRandomKeysTestFixture fixture; +}; + +TYPED_TEST_CASE_P(BlockStoreWithRandomKeysTest); + +TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndSameSize) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->create("mykey", 1024); + auto block2 = blockStore->create("mykey", 1024); + EXPECT_TRUE((bool)block); + EXPECT_FALSE((bool)block2); +} + +TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndDifferentSize) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->create("mykey", 1024); + auto block2 = blockStore->create("mykey", 4096); + EXPECT_TRUE((bool)block); + EXPECT_FALSE((bool)block2); +} + +TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndFirstNullSize) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->create("mykey", 0); + auto block2 = blockStore->create("mykey", 1024); + EXPECT_TRUE((bool)block); + EXPECT_FALSE((bool)block2); +} + +TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndSecondNullSize) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->create("mykey", 1024); + auto block2 = blockStore->create("mykey", 0); + EXPECT_TRUE((bool)block); + EXPECT_FALSE((bool)block2); +} + +TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndBothNullSize) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->create("mykey", 0); + auto block2 = blockStore->create("mykey", 0); + EXPECT_TRUE((bool)block); + EXPECT_FALSE((bool)block2); +} + +REGISTER_TYPED_TEST_CASE_P(BlockStoreWithRandomKeysTest, + CreateTwoBlocksWithSameKeyAndSameSize, + CreateTwoBlocksWithSameKeyAndDifferentSize, + CreateTwoBlocksWithSameKeyAndFirstNullSize, + CreateTwoBlocksWithSameKeyAndSecondNullSize, + CreateTwoBlocksWithSameKeyAndBothNullSize +); + + +#endif diff --git a/src/test/blockstore/utils/DataTest.cpp b/src/test/blockstore/utils/DataTest.cpp new file mode 100644 index 00000000..3060d472 --- /dev/null +++ b/src/test/blockstore/utils/DataTest.cpp @@ -0,0 +1,171 @@ +#include +#include +#include +#include "gtest/gtest.h" + +#include "test/testutils/TempFile.h" + +#include + +using ::testing::Test; +using ::testing::WithParamInterface; +using ::testing::Values; + +using std::ifstream; +using std::ofstream; + +using namespace blockstore; + +class DataTest: public Test { +public: + bool DataIsZeroes(const Data &data) { + for (size_t i = 0; i != data.size(); ++ i) { + if (((char*)data.data())[i] != 0) { + return false; + } + } + return true; + } + + void FillData(const DataBlockFixture &fillData, Data *data) { + ASSERT_EQ(fillData.size(), data->size()); + std::memcpy(data->data(), fillData.data(), fillData.size()); + } + + void EXPECT_DATA_CORRECT(const DataBlockFixture &expectedData, const Data &data) { + ASSERT_EQ(expectedData.size(), data.size()); + EXPECT_EQ(0, std::memcmp(expectedData.data(), data.data(), expectedData.size())); + } +}; + +class DataTestWithSizeParam: public DataTest, public WithParamInterface { +public: + DataBlockFixture randomData; + + DataTestWithSizeParam(): randomData(GetParam()) {} + + void FillData(Data *data) { + DataTest::FillData(randomData, data); + } + + void StoreData(const bf::path &filepath) { + ofstream file(filepath.c_str(), std::ios::binary | std::ios::trunc); + file.write(randomData.data(), randomData.size()); + } + + void EXPECT_STORED_FILE_DATA_CORRECT(const bf::path &filepath) { + EXPECT_EQ(randomData.size(), bf::file_size(filepath)); + + ifstream file(filepath.c_str(), std::ios::binary); + char *read_data = new char[randomData.size()]; + file.read(read_data, randomData.size()); + + EXPECT_EQ(0, std::memcmp(randomData.data(), read_data, randomData.size())); + delete[] read_data; + } + + void EXPECT_DATA_CORRECT(const Data &data) { + DataTest::EXPECT_DATA_CORRECT(randomData, data); + } +}; + +INSTANTIATE_TEST_CASE_P(DataTestWithSizeParam, DataTestWithSizeParam, Values(0, 1, 2, 1024, 4096, 10*1024*1024)); + +// Working on a large data area without a crash is a good indicator that we +// are actually working on memory that was validly allocated for us. +TEST_P(DataTestWithSizeParam, WriteAndCheck) { + Data data(GetParam()); + + FillData(&data); + EXPECT_DATA_CORRECT(data); +} + +TEST_P(DataTestWithSizeParam, Size) { + Data data(GetParam()); + EXPECT_EQ(GetParam(), data.size()); +} + +TEST_P(DataTestWithSizeParam, CheckStoredFile) { + Data data(GetParam()); + FillData(&data); + + TempFile file; + data.StoreToFile(file.path()); + + EXPECT_STORED_FILE_DATA_CORRECT(file.path()); +} + +TEST_P(DataTestWithSizeParam, CheckLoadedData) { + TempFile file; + StoreData(file.path()); + + Data data = Data::LoadFromFile(file.path()); + + EXPECT_DATA_CORRECT(data); +} + +TEST_P(DataTestWithSizeParam, StoreDoesntChangeData) { + Data data(GetParam()); + FillData(&data); + + TempFile file; + data.StoreToFile(file.path()); + + EXPECT_DATA_CORRECT(data); +} + +TEST_P(DataTestWithSizeParam, StoreAndLoad) { + Data data(GetParam()); + FillData(&data); + + TempFile file; + data.StoreToFile(file.path()); + Data loaded_data = Data::LoadFromFile(file.path()); + + EXPECT_DATA_CORRECT(loaded_data); +} + +TEST_F(DataTest, InitializeWithZeroes) { + Data data(10*1024); + data.FillWithZeroes(); + EXPECT_TRUE(DataIsZeroes(data)); +} + +TEST_F(DataTest, FillModifiedDataWithZeroes) { + Data data(10*1024); + DataBlockFixture randomData(10*1024); + FillData(randomData, &data); + EXPECT_FALSE(DataIsZeroes(data)); + + data.FillWithZeroes(); + EXPECT_TRUE(DataIsZeroes(data)); +} + +//Needs 64bit for representation. This value isn't in the size param list, because the list is also used for read/write checks. +TEST_F(DataTest, LargesizeSize) { + size_t size = 10L*1024*1024*1024; + Data data(size); + EXPECT_EQ(size, data.size()); +} + +// This test doesn't ensure that the Data class gives the memory region free, +// but it is a good indicator. +TEST_F(DataTest, InaccessibleAfterDeletion) { + Data *data = new Data(1); + ((char*)data->data())[0] = 0x3E; // Access data byte 0 + + delete data; + + EXPECT_DEATH( + ((char*)data->data())[0] = 0x3E, + "" + ); +} + +TEST_F(DataTest, LoadingNonexistingFile) { + TempFile file(false); // Pass false to constructor, so the tempfile is not created + EXPECT_THROW( + Data::LoadFromFile(file.path()), + FileDoesntExistException + ); +} diff --git a/src/test/blockstore/utils/RandomKeyGeneratorTest.cpp b/src/test/blockstore/utils/RandomKeyGeneratorTest.cpp new file mode 100644 index 00000000..644675df --- /dev/null +++ b/src/test/blockstore/utils/RandomKeyGeneratorTest.cpp @@ -0,0 +1,20 @@ +#include +#include "gtest/gtest.h" + + +using ::testing::Test; + +using std::string; + +using namespace blockstore; + +class RandomKeyGeneratorTest: public Test {}; + +TEST_F(RandomKeyGeneratorTest, RunsWithoutCrashes) { + string result = RandomKeyGenerator::singleton().create(); +} + +TEST_F(RandomKeyGeneratorTest, KeySizeIsAsSpecified) { + string result = RandomKeyGenerator::singleton().create(); + EXPECT_EQ(RandomKeyGenerator::KEYLENGTH, result.size()); +}