diff --git a/.gitignore b/.gitignore index e61aba92..9af772af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /build /cmake /.idea + diff --git a/.travis.yml b/.travis.yml index 21723057..fa9f9eda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,5 +35,6 @@ script: - ./test/cpp-utils/cpp-utils-test - ./run_with_fuse.sh ./test/fspp/fspp-test - ./test/parallelaccessstore/parallelaccessstore-test +- ./test/blockstore/blockstore-test after_script: - rm run_with_fuse.sh diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 56994ae3..882d5f74 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,5 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) add_subdirectory(cpp-utils) -add_subdirectory(fspp) \ No newline at end of file +add_subdirectory(fspp) +add_subdirectory(blockstore) \ No newline at end of file diff --git a/src/blockstore/CMakeLists.txt b/src/blockstore/CMakeLists.txt new file mode 100644 index 00000000..6adea221 --- /dev/null +++ b/src/blockstore/CMakeLists.txt @@ -0,0 +1,41 @@ +project (blockstore) + +set(SOURCES + utils/Key.cpp + utils/BlockStoreUtils.cpp + utils/FileDoesntExistException.cpp + interface/helpers/BlockStoreWithRandomKeys.cpp + implementations/testfake/FakeBlockStore.cpp + implementations/testfake/FakeBlock.cpp + implementations/inmemory/InMemoryBlock.cpp + implementations/inmemory/InMemoryBlockStore.cpp + implementations/parallelaccess/ParallelAccessBlockStore.cpp + implementations/parallelaccess/BlockRef.cpp + implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp + implementations/compressing/CompressingBlockStore.cpp + implementations/compressing/CompressedBlock.cpp + implementations/compressing/compressors/RunLengthEncoding.cpp + implementations/compressing/compressors/Gzip.cpp + implementations/encrypted/EncryptedBlockStore.cpp + implementations/encrypted/EncryptedBlock.cpp + implementations/ondisk/OnDiskBlockStore.cpp + implementations/ondisk/OnDiskBlock.cpp + implementations/caching/CachingBlockStore.cpp + implementations/caching/cache/PeriodicTask.cpp + implementations/caching/cache/CacheEntry.cpp + implementations/caching/cache/Cache.cpp + implementations/caching/cache/QueueMap.cpp + implementations/caching/CachedBlock.cpp + implementations/caching/NewBlock.cpp +) + +add_library(${PROJECT_NAME} STATIC ${SOURCES}) + +# This is needed by boost thread +if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") + target_link_libraries(${PROJECT_NAME} PRIVATE rt) +endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux") + +target_add_boost(${PROJECT_NAME} filesystem system thread) +target_enable_style_warnings(${PROJECT_NAME}) +target_activate_cpp14(${PROJECT_NAME}) diff --git a/src/blockstore/implementations/caching/CachedBlock.cpp b/src/blockstore/implementations/caching/CachedBlock.cpp new file mode 100644 index 00000000..8397888e --- /dev/null +++ b/src/blockstore/implementations/caching/CachedBlock.cpp @@ -0,0 +1,46 @@ +#include "CachedBlock.h" +#include "CachingBlockStore.h" + +using cpputils::unique_ref; + +namespace blockstore { +namespace caching { + +CachedBlock::CachedBlock(unique_ref baseBlock, CachingBlockStore *blockStore) + :Block(baseBlock->key()), + _blockStore(blockStore), + _baseBlock(std::move(baseBlock)) { +} + +CachedBlock::~CachedBlock() { + if (_baseBlock.get() != nullptr) { + _blockStore->release(std::move(_baseBlock)); + } +} + +const void *CachedBlock::data() const { + return _baseBlock->data(); +} + +void CachedBlock::write(const void *source, uint64_t offset, uint64_t size) { + return _baseBlock->write(source, offset, size); +} + +void CachedBlock::flush() { + return _baseBlock->flush(); +} + +size_t CachedBlock::size() const { + return _baseBlock->size(); +} + +void CachedBlock::resize(size_t newSize) { + return _baseBlock->resize(newSize); +} + +unique_ref CachedBlock::releaseBlock() { + return std::move(_baseBlock); +} + +} +} diff --git a/src/blockstore/implementations/caching/CachedBlock.h b/src/blockstore/implementations/caching/CachedBlock.h new file mode 100644 index 00000000..bcad16c6 --- /dev/null +++ b/src/blockstore/implementations/caching/CachedBlock.h @@ -0,0 +1,39 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_ + +#include "../../interface/Block.h" + +#include + +namespace blockstore { +namespace caching { +class CachingBlockStore; + +class CachedBlock final: public Block { +public: + //TODO Storing key twice (in parent class and in object pointed to). Once would be enough. + CachedBlock(cpputils::unique_ref baseBlock, CachingBlockStore *blockStore); + ~CachedBlock(); + + const void *data() const override; + void write(const void *source, uint64_t offset, uint64_t size) override; + void flush() override; + + size_t size() const override; + + void resize(size_t newSize) override; + + cpputils::unique_ref releaseBlock(); + +private: + CachingBlockStore *_blockStore; + cpputils::unique_ref _baseBlock; + + DISALLOW_COPY_AND_ASSIGN(CachedBlock); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/caching/CachingBlockStore.cpp b/src/blockstore/implementations/caching/CachingBlockStore.cpp new file mode 100644 index 00000000..055796d4 --- /dev/null +++ b/src/blockstore/implementations/caching/CachingBlockStore.cpp @@ -0,0 +1,92 @@ +#include "CachedBlock.h" +#include "NewBlock.h" +#include "CachingBlockStore.h" +#include "../../interface/Block.h" + +#include +#include +#include + +using cpputils::dynamic_pointer_move; +using cpputils::Data; +using boost::optional; +using cpputils::unique_ref; +using cpputils::make_unique_ref; +using boost::none; + +namespace blockstore { +namespace caching { + +CachingBlockStore::CachingBlockStore(cpputils::unique_ref baseBlockStore) + :_baseBlockStore(std::move(baseBlockStore)), _cache(), _numNewBlocks(0) { +} + +Key CachingBlockStore::createKey() { + return _baseBlockStore->createKey(); +} + +optional> CachingBlockStore::tryCreate(const Key &key, Data data) { + ASSERT(_cache.pop(key) == none, "Key already exists in cache"); + //TODO Shouldn't we return boost::none if the key already exists? + //TODO Key can also already exist but not be in the cache right now. + ++_numNewBlocks; + return unique_ref(make_unique_ref(make_unique_ref(key, std::move(data), this), this)); +} + +optional> CachingBlockStore::load(const Key &key) { + optional> optBlock = _cache.pop(key); + //TODO an optional<> class with .getOrElse() would make this code simpler. boost::optional<>::value_or_eval didn't seem to work with unique_ptr members. + if (optBlock != none) { + return optional>(make_unique_ref(std::move(*optBlock), this)); + } else { + auto block = _baseBlockStore->load(key); + if (block == none) { + return none; + } else { + return optional>(make_unique_ref(std::move(*block), this)); + } + } +} + +void CachingBlockStore::remove(cpputils::unique_ref block) { + auto cached_block = dynamic_pointer_move(block); + ASSERT(cached_block != none, "Passed block is not a CachedBlock"); + auto baseBlock = (*cached_block)->releaseBlock(); + auto baseNewBlock = dynamic_pointer_move(baseBlock); + if (baseNewBlock != none) { + if(!(*baseNewBlock)->alreadyExistsInBaseStore()) { + --_numNewBlocks; + } + (*baseNewBlock)->remove(); + } else { + _baseBlockStore->remove(std::move(baseBlock)); + } +} + +uint64_t CachingBlockStore::numBlocks() const { + return _baseBlockStore->numBlocks() + _numNewBlocks; +} + +void CachingBlockStore::release(unique_ref block) { + Key key = block->key(); + _cache.push(key, std::move(block)); +} + +optional> CachingBlockStore::tryCreateInBaseStore(const Key &key, Data data) { + auto block = _baseBlockStore->tryCreate(key, std::move(data)); + if (block != none) { + --_numNewBlocks; + } + return block; +} + +void CachingBlockStore::removeFromBaseStore(cpputils::unique_ref block) { + _baseBlockStore->remove(std::move(block)); +} + +void CachingBlockStore::flush() { + _cache.flush(); +} + +} +} diff --git a/src/blockstore/implementations/caching/CachingBlockStore.h b/src/blockstore/implementations/caching/CachingBlockStore.h new file mode 100644 index 00000000..7ec02af1 --- /dev/null +++ b/src/blockstore/implementations/caching/CachingBlockStore.h @@ -0,0 +1,40 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_ + +#include "cache/Cache.h" +#include "../../interface/BlockStore.h" + +namespace blockstore { +namespace caching { + +//TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel) +class CachingBlockStore final: public BlockStore { +public: + explicit CachingBlockStore(cpputils::unique_ref baseBlockStore); + + Key createKey() override; + boost::optional> tryCreate(const Key &key, cpputils::Data data) override; + boost::optional> load(const Key &key) override; + void remove(cpputils::unique_ref block) override; + uint64_t numBlocks() const override; + + void release(cpputils::unique_ref block); + + boost::optional> tryCreateInBaseStore(const Key &key, cpputils::Data data); + void removeFromBaseStore(cpputils::unique_ref block); + + void flush(); + +private: + cpputils::unique_ref _baseBlockStore; + Cache, 1000> _cache; + uint32_t _numNewBlocks; + + DISALLOW_COPY_AND_ASSIGN(CachingBlockStore); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/caching/NewBlock.cpp b/src/blockstore/implementations/caching/NewBlock.cpp new file mode 100644 index 00000000..18bf8425 --- /dev/null +++ b/src/blockstore/implementations/caching/NewBlock.cpp @@ -0,0 +1,75 @@ +#include "NewBlock.h" +#include "CachingBlockStore.h" +#include +#include + +using cpputils::Data; +using boost::none; + +namespace blockstore { +namespace caching { + +NewBlock::NewBlock(const Key &key, Data data, CachingBlockStore *blockStore) + :Block(key), + _blockStore(blockStore), + _data(std::move(data)), + _baseBlock(none), + _dataChanged(true) { +} + +NewBlock::~NewBlock() { + writeToBaseBlockIfChanged(); +} + +const void *NewBlock::data() const { + return _data.data(); +} + +void NewBlock::write(const void *source, uint64_t offset, uint64_t size) { + ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area"); + std::memcpy((uint8_t*)_data.data()+offset, source, size); + _dataChanged = true; +} + +void NewBlock::writeToBaseBlockIfChanged() { + if (_dataChanged) { + if (_baseBlock == none) { + //TODO _data.copy() necessary? + auto newBase = _blockStore->tryCreateInBaseStore(key(), _data.copy()); + ASSERT(newBase != boost::none, "Couldn't create base block"); //TODO What if tryCreate fails due to a duplicate key? We should ensure we don't use duplicate keys. + _baseBlock = std::move(*newBase); + } else { + (*_baseBlock)->write(_data.data(), 0, _data.size()); + } + _dataChanged = false; + } +} + +void NewBlock::remove() { + if (_baseBlock != none) { + _blockStore->removeFromBaseStore(std::move(*_baseBlock)); + } + _dataChanged = false; +} + +void NewBlock::flush() { + writeToBaseBlockIfChanged(); + ASSERT(_baseBlock != none, "At this point, the base block should already have been created but wasn't"); + (*_baseBlock)->flush(); +} + +size_t NewBlock::size() const { + return _data.size(); +} + +void NewBlock::resize(size_t newSize) { + _data = cpputils::DataUtils::resize(std::move(_data), newSize); + _dataChanged = true; +} + +bool NewBlock::alreadyExistsInBaseStore() const { + return _baseBlock != none; +} + +} +} diff --git a/src/blockstore/implementations/caching/NewBlock.h b/src/blockstore/implementations/caching/NewBlock.h new file mode 100644 index 00000000..84929204 --- /dev/null +++ b/src/blockstore/implementations/caching/NewBlock.h @@ -0,0 +1,52 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_ + +#include "../../interface/BlockStore.h" +#include + +#include "messmer/cpp-utils/macros.h" +#include + +namespace blockstore { +namespace caching { +class CachingBlockStore; + +//TODO Does it make sense to write a general DataBackedBlock that just stores a Data object and maps the block operations to it? +// Can we reuse that object somewhere else? +// Maybe a second abstract class for BlockRefBackedBlock? + +// This is a block that was created in CachingBlockStore, but doesn't exist in the base block store yet. +// It only exists in the cache and it is created in the base block store when destructed. +class NewBlock final: public Block { +public: + NewBlock(const Key &key, cpputils::Data data, CachingBlockStore *blockStore); + ~NewBlock(); + + const void *data() const override; + void write(const void *source, uint64_t offset, uint64_t size) override; + void flush() override; + + size_t size() const override; + + void resize(size_t newSize) override; + + void remove(); + + bool alreadyExistsInBaseStore() const; + +private: + CachingBlockStore *_blockStore; + cpputils::Data _data; + boost::optional> _baseBlock; + bool _dataChanged; + + void writeToBaseBlockIfChanged(); + + DISALLOW_COPY_AND_ASSIGN(NewBlock); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/caching/cache/Cache.cpp b/src/blockstore/implementations/caching/cache/Cache.cpp new file mode 100644 index 00000000..2d464949 --- /dev/null +++ b/src/blockstore/implementations/caching/cache/Cache.cpp @@ -0,0 +1 @@ +#include "Cache.h" diff --git a/src/blockstore/implementations/caching/cache/Cache.h b/src/blockstore/implementations/caching/cache/Cache.h new file mode 100644 index 00000000..b20c34d9 --- /dev/null +++ b/src/blockstore/implementations/caching/cache/Cache.h @@ -0,0 +1,178 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHE_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHE_H_ + +#include "CacheEntry.h" +#include "QueueMap.h" +#include "PeriodicTask.h" +#include +#include +#include +#include +#include +#include + +namespace blockstore { +namespace caching { + +template +class Cache final { +public: + //TODO Current MAX_LIFETIME_SEC only considers time since the element was last pushed to the Cache. Also insert a real MAX_LIFETIME_SEC that forces resync of entries that have been pushed/popped often (e.g. the root blob) + //TODO Experiment with good values + static constexpr double PURGE_LIFETIME_SEC = 0.5; //When an entry has this age, it will be purged from the cache + static constexpr double PURGE_INTERVAL = 0.5; // With this interval, we check for entries to purge + static constexpr double MAX_LIFETIME_SEC = PURGE_LIFETIME_SEC + PURGE_INTERVAL; // This is the oldest age an entry can reach (given purging works in an ideal world, i.e. with the ideal interval and in zero time) + + Cache(); + ~Cache(); + + uint32_t size() const; + + void push(const Key &key, Value value); + boost::optional pop(const Key &key); + + void flush(); + +private: + void _makeSpaceForEntry(std::unique_lock *lock); + void _deleteEntry(std::unique_lock *lock); + void _deleteOldEntriesParallel(); + void _deleteAllEntriesParallel(); + void _deleteMatchingEntriesAtBeginningParallel(std::function &)> matches); + void _deleteMatchingEntriesAtBeginning(std::function &)> matches); + bool _deleteMatchingEntryAtBeginning(std::function &)> matches); + + mutable std::mutex _mutex; + cpputils::LockPool _currentlyFlushingEntries; + QueueMap> _cachedBlocks; + std::unique_ptr _timeoutFlusher; + + DISALLOW_COPY_AND_ASSIGN(Cache); +}; + +template constexpr double Cache::PURGE_LIFETIME_SEC; +template constexpr double Cache::PURGE_INTERVAL; +template constexpr double Cache::MAX_LIFETIME_SEC; + +template +Cache::Cache(): _mutex(), _currentlyFlushingEntries(), _cachedBlocks(), _timeoutFlusher(nullptr) { + //Don't initialize timeoutFlusher in the initializer list, + //because it then might already call Cache::popOldEntries() before Cache is done constructing. + _timeoutFlusher = std::make_unique(std::bind(&Cache::_deleteOldEntriesParallel, this), PURGE_INTERVAL); +} + +template +Cache::~Cache() { + _deleteAllEntriesParallel(); + ASSERT(_cachedBlocks.size() == 0, "Error in _deleteAllEntriesParallel()"); +} + +template +boost::optional Cache::pop(const Key &key) { + std::unique_lock lock(_mutex); + cpputils::MutexPoolLock lockEntryFromBeingPopped(&_currentlyFlushingEntries, key, &lock); + + auto found = _cachedBlocks.pop(key); + if (!found) { + return boost::none; + } + return found->releaseValue(); +} + +template +void Cache::push(const Key &key, Value value) { + std::unique_lock lock(_mutex); + ASSERT(_cachedBlocks.size() <= MAX_ENTRIES, "Cache too full"); + _makeSpaceForEntry(&lock); + _cachedBlocks.push(key, CacheEntry(std::move(value))); +} + +template +void Cache::_makeSpaceForEntry(std::unique_lock *lock) { + // _deleteEntry releases the lock while the Value destructor is running. + // So we can destruct multiple entries in parallel and also call pop() or push() while doing so. + // However, if another thread calls push() before we get the lock back, the cache is full again. + // That's why we need the while() loop here. + while (_cachedBlocks.size() == MAX_ENTRIES) { + _deleteEntry(lock); + } + ASSERT(_cachedBlocks.size() < MAX_ENTRIES, "Removing entry from cache didn't work"); +}; + +template +void Cache::_deleteEntry(std::unique_lock *lock) { + auto key = _cachedBlocks.peekKey(); + ASSERT(key != boost::none, "There was no entry to delete"); + cpputils::MutexPoolLock lockEntryFromBeingPopped(&_currentlyFlushingEntries, *key); + auto value = _cachedBlocks.pop(); + // Call destructor outside of the unique_lock, + // i.e. pop() and push() can be called here, except for pop() on the element in _currentlyFlushingEntries + lock->unlock(); + value = boost::none; // Call destructor + lock->lock(); +}; + +template +void Cache::_deleteAllEntriesParallel() { + return _deleteMatchingEntriesAtBeginningParallel([] (const CacheEntry &) { + return true; + }); +} + +template +void Cache::_deleteOldEntriesParallel() { + return _deleteMatchingEntriesAtBeginningParallel([] (const CacheEntry &entry) { + return entry.ageSeconds() > PURGE_LIFETIME_SEC; + }); +} + +template +void Cache::_deleteMatchingEntriesAtBeginningParallel(std::function &)> matches) { + // Twice the number of cores, so we use full CPU even if half the threads are doing I/O + unsigned int numThreads = 2 * std::max(1u, std::thread::hardware_concurrency()); + std::vector> waitHandles; + for (unsigned int i = 0; i < numThreads; ++i) { + waitHandles.push_back(std::async(std::launch::async, [this, matches] { + _deleteMatchingEntriesAtBeginning(matches); + })); + } + for (auto & waitHandle : waitHandles) { + waitHandle.wait(); + } +}; + +template +void Cache::_deleteMatchingEntriesAtBeginning(std::function &)> matches) { + while (_deleteMatchingEntryAtBeginning(matches)) {} +} + +template +bool Cache::_deleteMatchingEntryAtBeginning(std::function &)> matches) { + // This function can be called in parallel by multiple threads and will then cause the Value destructors + // to be called in parallel. The call to _deleteEntry() releases the lock while the Value destructor is running. + std::unique_lock lock(_mutex); + if (_cachedBlocks.size() > 0 && matches(*_cachedBlocks.peek())) { + _deleteEntry(&lock); + return true; + } else { + return false; + } +}; + +template +uint32_t Cache::size() const { + std::unique_lock lock(_mutex); + return _cachedBlocks.size(); +}; + +template +void Cache::flush() { + //TODO Test flush() + return _deleteAllEntriesParallel(); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/caching/cache/CacheEntry.cpp b/src/blockstore/implementations/caching/cache/CacheEntry.cpp new file mode 100644 index 00000000..a3420ac8 --- /dev/null +++ b/src/blockstore/implementations/caching/cache/CacheEntry.cpp @@ -0,0 +1 @@ +#include "CacheEntry.h" diff --git a/src/blockstore/implementations/caching/cache/CacheEntry.h b/src/blockstore/implementations/caching/cache/CacheEntry.h new file mode 100644 index 00000000..56d77cdb --- /dev/null +++ b/src/blockstore/implementations/caching/cache/CacheEntry.h @@ -0,0 +1,44 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHEENTRY_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_CACHEENTRY_H_ + +#include +#include +#include +#include + +namespace blockstore { +namespace caching { + +template +class CacheEntry final { +public: + explicit CacheEntry(Value value): _lastAccess(currentTime()), _value(std::move(value)) { + } + + CacheEntry(CacheEntry &&) = default; + + double ageSeconds() const { + return ((double)(currentTime() - _lastAccess).total_nanoseconds()) / ((double)1000000000); + } + + Value releaseValue() { + return std::move(_value); + } + +private: + boost::posix_time::ptime _lastAccess; + Value _value; + + static boost::posix_time::ptime currentTime() { + return boost::posix_time::microsec_clock::local_time(); + } + + DISALLOW_COPY_AND_ASSIGN(CacheEntry); +}; + +} +} + + +#endif diff --git a/src/blockstore/implementations/caching/cache/PeriodicTask.cpp b/src/blockstore/implementations/caching/cache/PeriodicTask.cpp new file mode 100644 index 00000000..14af6d5e --- /dev/null +++ b/src/blockstore/implementations/caching/cache/PeriodicTask.cpp @@ -0,0 +1,27 @@ +#include "PeriodicTask.h" +#include + +using std::function; +using std::endl; +using namespace cpputils::logging; + +namespace blockstore { +namespace caching { + +PeriodicTask::PeriodicTask(function task, double intervalSec) : + _task(task), + _interval((uint64_t)(UINT64_C(1000000000) * intervalSec)), + _thread(std::bind(&PeriodicTask::_loopIteration, this)) { + _thread.start(); +} + +bool PeriodicTask::_loopIteration() { + //Has to be boost::this_thread::sleep_for and not std::this_thread::sleep_for, because it has to be interruptible. + //LoopThread will interrupt this method if it has to be restarted. + boost::this_thread::sleep_for(_interval); + _task(); + return true; // Run another iteration (don't terminate thread) +} + +} +} diff --git a/src/blockstore/implementations/caching/cache/PeriodicTask.h b/src/blockstore/implementations/caching/cache/PeriodicTask.h new file mode 100644 index 00000000..3a8fd8a2 --- /dev/null +++ b/src/blockstore/implementations/caching/cache/PeriodicTask.h @@ -0,0 +1,32 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_PERIODICTASK_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_PERIODICTASK_H_ + +#include +#include +#include + +namespace blockstore { +namespace caching { + +class PeriodicTask final { +public: + PeriodicTask(std::function task, double intervalSec); + +private: + bool _loopIteration(); + + std::function _task; + boost::chrono::nanoseconds _interval; + + //This member has to be last, so the thread is destructed first. Otherwise the thread might access elements from a + //partly destructed PeriodicTask. + cpputils::LoopThread _thread; + + DISALLOW_COPY_AND_ASSIGN(PeriodicTask); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/caching/cache/QueueMap.cpp b/src/blockstore/implementations/caching/cache/QueueMap.cpp new file mode 100644 index 00000000..510d8ece --- /dev/null +++ b/src/blockstore/implementations/caching/cache/QueueMap.cpp @@ -0,0 +1 @@ +#include "QueueMap.h" diff --git a/src/blockstore/implementations/caching/cache/QueueMap.h b/src/blockstore/implementations/caching/cache/QueueMap.h new file mode 100644 index 00000000..43f4b1ee --- /dev/null +++ b/src/blockstore/implementations/caching/cache/QueueMap.h @@ -0,0 +1,121 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_QUEUEMAP_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_QUEUEMAP_H_ + +#include +#include +#include +#include +#include +#include + +namespace blockstore { +namespace caching { + +//TODO FreeList for performance (malloc is expensive) +//TODO Single linked list with pointer to last element (for insertion) should be enough for a queue. No double linked list needed. +// But then, popping arbitrary elements needs to be rewritten so that _removeFromQueue() is _removeSuccessorFromQueue() +// and the map doesn't store the element itself, but its predecessor. That is, popping might be a bit slower. Test with experiments! + +// A class that is a queue and a map at the same time. We could also see it as an addressable queue. +template +class QueueMap final { +public: + QueueMap(): _entries(), _sentinel(&_sentinel, &_sentinel) { + } + ~QueueMap() { + for (auto &entry : _entries) { + entry.second.release(); + } + } + + void push(const Key &key, Value value) { + auto newEntry = _entries.emplace(std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple(_sentinel.prev, &_sentinel)); + if (!newEntry.second) { + throw std::logic_error("There is already an element with this key"); + } + newEntry.first->second.init(&newEntry.first->first, std::move(value)); + //The following is ok, because std::unordered_map never invalidates pointers to its entries + _sentinel.prev->next = &newEntry.first->second; + _sentinel.prev = &newEntry.first->second; + } + + boost::optional pop(const Key &key) { + auto found = _entries.find(key); + if (found == _entries.end()) { + return boost::none; + } + _removeFromQueue(found->second); + auto value = found->second.release(); + _entries.erase(found); + return std::move(value); + } + + boost::optional pop() { + if(_sentinel.next == &_sentinel) { + return boost::none; + } + return pop(*_sentinel.next->key); + } + + boost::optional peekKey() { + if(_sentinel.next == &_sentinel) { + return boost::none; + } + return *_sentinel.next->key; + } + + boost::optional peek() { + if(_sentinel.next == &_sentinel) { + return boost::none; + } + return _sentinel.next->value(); + } + + uint32_t size() const { + return _entries.size(); + } + +private: + class Entry final { + public: + Entry(Entry *prev_, Entry *next_): prev(prev_), next(next_), key(nullptr), __value() { + } + void init(const Key *key_, Value value_) { + key = key_; + new(__value) Value(std::move(value_)); + } + Value release() { + Value value = std::move(*_value()); + _value()->~Value(); + return value; + } + const Value &value() { + return *_value(); + } + Entry *prev; + Entry *next; + const Key *key; + private: + Value *_value() { + return reinterpret_cast(__value); + } + alignas(Value) char __value[sizeof(Value)]; + DISALLOW_COPY_AND_ASSIGN(Entry); + }; + + void _removeFromQueue(const Entry &entry) { + entry.prev->next = entry.next; + entry.next->prev = entry.prev; + } + + std::unordered_map _entries; + Entry _sentinel; + + DISALLOW_COPY_AND_ASSIGN(QueueMap); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/compressing/CompressedBlock.cpp b/src/blockstore/implementations/compressing/CompressedBlock.cpp new file mode 100644 index 00000000..e226e7e8 --- /dev/null +++ b/src/blockstore/implementations/compressing/CompressedBlock.cpp @@ -0,0 +1 @@ +#include "CompressedBlock.h" diff --git a/src/blockstore/implementations/compressing/CompressedBlock.h b/src/blockstore/implementations/compressing/CompressedBlock.h new file mode 100644 index 00000000..cf7b2301 --- /dev/null +++ b/src/blockstore/implementations/compressing/CompressedBlock.h @@ -0,0 +1,127 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSEDBLOCK_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSEDBLOCK_H_ + +#include "../../interface/Block.h" +#include "../../interface/BlockStore.h" +#include +#include +#include + +namespace blockstore { +class BlockStore; +namespace compressing { +template class CompressingBlockStore; + +template +class CompressedBlock final: public Block { +public: + static boost::optional> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData); + static cpputils::unique_ref Decompress(cpputils::unique_ref baseBlock); + + CompressedBlock(cpputils::unique_ref baseBlock, cpputils::Data decompressedData); + ~CompressedBlock(); + + const void *data() const override; + void write(const void *source, uint64_t offset, uint64_t size) override; + + void flush() override; + + size_t size() const override; + void resize(size_t newSize) override; + + cpputils::unique_ref releaseBaseBlock(); + +private: + void _compressToBaseBlock(); + + cpputils::unique_ref _baseBlock; + cpputils::Data _decompressedData; + std::mutex _mutex; + bool _dataChanged; + + DISALLOW_COPY_AND_ASSIGN(CompressedBlock); +}; + +template +boost::optional>> CompressedBlock::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData) { + cpputils::Data compressed = Compressor::Compress(decompressedData); + auto baseBlock = baseBlockStore->tryCreate(key, std::move(compressed)); + if (baseBlock == boost::none) { + //TODO Test this code branch + return boost::none; + } + + return cpputils::make_unique_ref>(std::move(*baseBlock), std::move(decompressedData)); +} + +template +cpputils::unique_ref> CompressedBlock::Decompress(cpputils::unique_ref baseBlock) { + cpputils::Data decompressed = Compressor::Decompress((byte*)baseBlock->data(), baseBlock->size()); + return cpputils::make_unique_ref>(std::move(baseBlock), std::move(decompressed)); +} + +template +CompressedBlock::CompressedBlock(cpputils::unique_ref baseBlock, cpputils::Data decompressedData) + : Block(baseBlock->key()), + _baseBlock(std::move(baseBlock)), + _decompressedData(std::move(decompressedData)), + _dataChanged(false) { +} + +template +CompressedBlock::~CompressedBlock() { + std::unique_lock lock(_mutex); + _compressToBaseBlock(); +} + +template +const void *CompressedBlock::data() const { + return _decompressedData.data(); +} + +template +void CompressedBlock::write(const void *source, uint64_t offset, uint64_t size) { + std::memcpy((uint8_t*)_decompressedData.dataOffset(offset), source, size); + _dataChanged = true; +} + +template +void CompressedBlock::flush() { + std::unique_lock lock(_mutex); + _compressToBaseBlock(); + return _baseBlock->flush(); +} + +template +size_t CompressedBlock::size() const { + return _decompressedData.size(); +} + +template +void CompressedBlock::resize(size_t newSize) { + _decompressedData = cpputils::DataUtils::resize(std::move(_decompressedData), newSize); + _dataChanged = true; +} + +template +cpputils::unique_ref CompressedBlock::releaseBaseBlock() { + std::unique_lock lock(_mutex); + _compressToBaseBlock(); + return std::move(_baseBlock); +} + +template +void CompressedBlock::_compressToBaseBlock() { + if (_dataChanged) { + cpputils::Data compressed = Compressor::Compress(_decompressedData); + _baseBlock->resize(compressed.size()); + _baseBlock->write(compressed.data(), 0, compressed.size()); + _dataChanged = false; + } +} + +} +} + +#endif diff --git a/src/blockstore/implementations/compressing/CompressingBlockStore.cpp b/src/blockstore/implementations/compressing/CompressingBlockStore.cpp new file mode 100644 index 00000000..ab79d260 --- /dev/null +++ b/src/blockstore/implementations/compressing/CompressingBlockStore.cpp @@ -0,0 +1 @@ +#include "CompressingBlockStore.h" diff --git a/src/blockstore/implementations/compressing/CompressingBlockStore.h b/src/blockstore/implementations/compressing/CompressingBlockStore.h new file mode 100644 index 00000000..06b22c66 --- /dev/null +++ b/src/blockstore/implementations/compressing/CompressingBlockStore.h @@ -0,0 +1,77 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSINGBLOCKSTORE_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSINGBLOCKSTORE_H_ + +#include "../../interface/BlockStore.h" +#include "CompressedBlock.h" + +namespace blockstore { +namespace compressing { + +template +class CompressingBlockStore final: public BlockStore { +public: + CompressingBlockStore(cpputils::unique_ref baseBlockStore); + ~CompressingBlockStore(); + + Key createKey() override; + boost::optional> tryCreate(const Key &key, cpputils::Data data) override; + boost::optional> load(const Key &key) override; + void remove(cpputils::unique_ref block) override; + uint64_t numBlocks() const override; + +private: + cpputils::unique_ref _baseBlockStore; + + DISALLOW_COPY_AND_ASSIGN(CompressingBlockStore); +}; + +template +CompressingBlockStore::CompressingBlockStore(cpputils::unique_ref baseBlockStore) + : _baseBlockStore(std::move(baseBlockStore)) { +} + +template +CompressingBlockStore::~CompressingBlockStore() { +} + +template +Key CompressingBlockStore::createKey() { + return _baseBlockStore->createKey(); +} + +template +boost::optional> CompressingBlockStore::tryCreate(const Key &key, cpputils::Data data) { + auto result = CompressedBlock::TryCreateNew(_baseBlockStore.get(), key, std::move(data)); + if (result == boost::none) { + return boost::none; + } + return cpputils::unique_ref(std::move(*result)); +} + +template +boost::optional> CompressingBlockStore::load(const Key &key) { + auto loaded = _baseBlockStore->load(key); + if (loaded == boost::none) { + return boost::none; + } + return boost::optional>(CompressedBlock::Decompress(std::move(*loaded))); +} + +template +void CompressingBlockStore::remove(cpputils::unique_ref block) { + auto _block = cpputils::dynamic_pointer_move>(block); + ASSERT(_block != boost::none, "Wrong block type"); + auto baseBlock = (*_block)->releaseBaseBlock(); + return _baseBlockStore->remove(std::move(baseBlock)); +} + +template +uint64_t CompressingBlockStore::numBlocks() const { + return _baseBlockStore->numBlocks(); +} + +} +} + +#endif diff --git a/src/blockstore/implementations/compressing/compressors/Gzip.cpp b/src/blockstore/implementations/compressing/compressors/Gzip.cpp new file mode 100644 index 00000000..4c2d59da --- /dev/null +++ b/src/blockstore/implementations/compressing/compressors/Gzip.cpp @@ -0,0 +1,29 @@ +#include "Gzip.h" +#include + +using cpputils::Data; + +namespace blockstore { + namespace compressing { + + Data Gzip::Compress(const Data &data) { + CryptoPP::Gzip zipper; + zipper.Put((byte *) data.data(), data.size()); + zipper.MessageEnd(); + Data compressed(zipper.MaxRetrievable()); + zipper.Get((byte *) compressed.data(), compressed.size()); + return compressed; + } + + Data Gzip::Decompress(const void *data, size_t size) { + //TODO Change interface to taking cpputils::Data objects (needs changing blockstore so we can read their "class Data", because this is called from CompressedBlock::Decompress()). + CryptoPP::Gunzip zipper; + zipper.Put((byte *) data, size); + zipper.MessageEnd(); + Data decompressed(zipper.MaxRetrievable()); + zipper.Get((byte *) decompressed.data(), decompressed.size()); + return decompressed; + } + + } +} \ No newline at end of file diff --git a/src/blockstore/implementations/compressing/compressors/Gzip.h b/src/blockstore/implementations/compressing/compressors/Gzip.h new file mode 100644 index 00000000..ab53e372 --- /dev/null +++ b/src/blockstore/implementations/compressing/compressors/Gzip.h @@ -0,0 +1,18 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_GZIP_H +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_GZIP_H + +#include + +namespace blockstore { + namespace compressing { + class Gzip { + public: + static cpputils::Data Compress(const cpputils::Data &data); + + static cpputils::Data Decompress(const void *data, size_t size); + }; + } +} + +#endif diff --git a/src/blockstore/implementations/compressing/compressors/RunLengthEncoding.cpp b/src/blockstore/implementations/compressing/compressors/RunLengthEncoding.cpp new file mode 100644 index 00000000..24e0501e --- /dev/null +++ b/src/blockstore/implementations/compressing/compressors/RunLengthEncoding.cpp @@ -0,0 +1,139 @@ +#include "RunLengthEncoding.h" +#include +#include + +using cpputils::Data; +using std::string; +using std::ostringstream; +using std::istringstream; + +namespace blockstore { + namespace compressing { + + // Alternatively store a run of arbitrary bytes and a run of identical bytes. + // Each run is preceded by its length. Length fields are uint16_t. + // Example: 2 - 5 - 8 - 10 - 3 - 0 - 2 - 0 + // Length 2 arbitrary bytes (values: 5, 8), the next 10 bytes store "3" each, + // then 0 arbitrary bytes and 2x "0". + + Data RunLengthEncoding::Compress(const Data &data) { + ostringstream compressed; + uint8_t *current = (uint8_t*)data.data(); + uint8_t *end = (uint8_t*)data.data()+data.size(); + while (current < end) { + _encodeArbitraryWords(¤t, end, &compressed); + ASSERT(current <= end, "Overflow"); + if (current == end) { + break; + } + _encodeIdenticalWords(¤t, end, &compressed); + ASSERT(current <= end, "Overflow"); + } + return _extractData(&compressed); + } + + void RunLengthEncoding::_encodeArbitraryWords(uint8_t **current, uint8_t* end, ostringstream *output) { + uint16_t size = _arbitraryRunLength(*current, end); + output->write((const char*)&size, sizeof(uint16_t)); + output->write((const char*)*current, size); + *current += size; + } + + uint16_t RunLengthEncoding::_arbitraryRunLength(uint8_t *start, uint8_t* end) { + // Each stopping of an arbitrary bytes run costs us 5 byte, because we have to store the length + // for the identical bytes run (2 byte), the identical byte itself (1 byte) and the length for the next arbitrary bytes run (2 byte). + // So to get an advantage from stopping an arbitrary bytes run, at least 6 bytes have to be identical. + + // realEnd avoids an overflow of the 16bit counter + uint8_t *realEnd = std::min(end, start + std::numeric_limits::max()); + + // Count the number of identical bytes and return if it finds a run of more than 6 identical bytes. + uint8_t lastByte = *start + 1; // Something different from the first byte + uint8_t numIdenticalBytes = 1; + for(uint8_t *current = start; current != realEnd; ++current) { + if (*current == lastByte) { + ++numIdenticalBytes; + if (numIdenticalBytes == 6) { + return current - start - 5; //-5, because the end pointer for the arbitrary byte run should point to the first identical byte, not the one before. + } + } else { + numIdenticalBytes = 1; + } + lastByte = *current; + } + //It wasn't worth stopping the arbitrary bytes run anywhere. The whole region should be an arbitrary run. + return realEnd-start; + } + + void RunLengthEncoding::_encodeIdenticalWords(uint8_t **current, uint8_t* end, ostringstream *output) { + uint16_t size = _countIdenticalBytes(*current, end); + output->write((const char*)&size, sizeof(uint16_t)); + output->write((const char*)*current, 1); + *current += size; + } + + uint16_t RunLengthEncoding::_countIdenticalBytes(uint8_t *start, uint8_t *end) { + uint8_t *realEnd = std::min(end, start + std::numeric_limits::max()); // This prevents overflow of the 16bit counter + for (uint8_t *current = start+1; current != realEnd; ++current) { + if (*current != *start) { + return current-start; + } + } + // All bytes have been identical + return realEnd - start; + } + + Data RunLengthEncoding::_extractData(ostringstream *stream) { + string str = stream->str(); + Data data(str.size()); + std::memcpy(data.data(), str.c_str(), str.size()); + return data; + } + + Data RunLengthEncoding::Decompress(const void *data, size_t size) { + istringstream stream; + _parseData((uint8_t*)data, size, &stream); + ostringstream decompressed; + while(_hasData(&stream)) { + _decodeArbitraryWords(&stream, &decompressed); + if (!_hasData(&stream)) { + break; + } + _decodeIdenticalWords(&stream, &decompressed); + } + return _extractData(&decompressed); + } + + bool RunLengthEncoding::_hasData(istringstream *str) { + str->peek(); + return !str->eof(); + } + + void RunLengthEncoding::_parseData(const uint8_t *data, size_t size, istringstream *result) { + result->str(string((const char*)data, size)); + } + + void RunLengthEncoding::_decodeArbitraryWords(istringstream *stream, ostringstream *decompressed) { + uint16_t size; + stream->read((char*)&size, sizeof(uint16_t)); + ASSERT(stream->good(), "Premature end of stream"); + Data run(size); + stream->read((char*)run.data(), size); + ASSERT(stream->good(), "Premature end of stream"); + decompressed->write((const char*)run.data(), run.size()); + } + + void RunLengthEncoding::_decodeIdenticalWords(istringstream *stream, ostringstream *decompressed) { + uint16_t size; + stream->read((char*)&size, sizeof(uint16_t)); + ASSERT(stream->good(), "Premature end of stream"); + uint8_t value; + stream->read((char*)&value, 1); + ASSERT(stream->good(), "Premature end of stream"); + Data run(size); + std::memset(run.data(), value, run.size()); + decompressed->write((const char*)run.data(), run.size()); + } + + } +} diff --git a/src/blockstore/implementations/compressing/compressors/RunLengthEncoding.h b/src/blockstore/implementations/compressing/compressors/RunLengthEncoding.h new file mode 100644 index 00000000..29a2c739 --- /dev/null +++ b/src/blockstore/implementations/compressing/compressors/RunLengthEncoding.h @@ -0,0 +1,29 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_RUNLENGTHENCODING_H +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_COMPRESSING_COMPRESSORS_RUNLENGTHENCODING_H + +#include + +namespace blockstore { + namespace compressing { + class RunLengthEncoding { + public: + static cpputils::Data Compress(const cpputils::Data &data); + + static cpputils::Data Decompress(const void *data, size_t size); + + private: + static void _encodeArbitraryWords(uint8_t **current, uint8_t* end, std::ostringstream *output); + static uint16_t _arbitraryRunLength(uint8_t *start, uint8_t* end); + static void _encodeIdenticalWords(uint8_t **current, uint8_t* end, std::ostringstream *output); + static uint16_t _countIdenticalBytes(uint8_t *start, uint8_t *end); + static bool _hasData(std::istringstream *stream); + static cpputils::Data _extractData(std::ostringstream *stream); + static void _parseData(const uint8_t *data, size_t size, std::istringstream *result); + static void _decodeArbitraryWords(std::istringstream *stream, std::ostringstream *decompressed); + static void _decodeIdenticalWords(std::istringstream *stream, std::ostringstream *decompressed); + }; + } +} + +#endif diff --git a/src/blockstore/implementations/encrypted/EncryptedBlock.cpp b/src/blockstore/implementations/encrypted/EncryptedBlock.cpp new file mode 100644 index 00000000..5f66d9c9 --- /dev/null +++ b/src/blockstore/implementations/encrypted/EncryptedBlock.cpp @@ -0,0 +1 @@ +#include "EncryptedBlock.h" diff --git a/src/blockstore/implementations/encrypted/EncryptedBlock.h b/src/blockstore/implementations/encrypted/EncryptedBlock.h new file mode 100644 index 00000000..5fbe8c2c --- /dev/null +++ b/src/blockstore/implementations/encrypted/EncryptedBlock.h @@ -0,0 +1,177 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_ + +#include "../../interface/Block.h" +#include +#include "../../interface/BlockStore.h" + +#include "messmer/cpp-utils/macros.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace blockstore { +namespace encrypted { +template class EncryptedBlockStore; + +//TODO Test EncryptedBlock + +//TODO Fix mutexes & locks (basically true for all blockstores) + +template +class EncryptedBlock final: public Block { +public: + BOOST_CONCEPT_ASSERT((cpputils::CipherConcept)); + static boost::optional> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey); + static boost::optional> TryDecrypt(cpputils::unique_ref baseBlock, const typename Cipher::EncryptionKey &key); + + //TODO Storing key twice (in parent class and in object pointed to). Once would be enough. + EncryptedBlock(cpputils::unique_ref baseBlock, const typename Cipher::EncryptionKey &key, cpputils::Data plaintextWithHeader); + ~EncryptedBlock(); + + const void *data() const override; + void write(const void *source, uint64_t offset, uint64_t count) override; + void flush() override; + + size_t size() const override; + void resize(size_t newSize) override; + + cpputils::unique_ref releaseBlock(); + +private: + cpputils::unique_ref _baseBlock; // TODO Do I need the ciphertext block in memory or is the key enough? + cpputils::Data _plaintextWithHeader; + typename Cipher::EncryptionKey _encKey; + bool _dataChanged; + + static constexpr unsigned int HEADER_LENGTH = Key::BINARY_LENGTH; + + void _encryptToBaseBlock(); + static cpputils::Data _prependKeyHeaderToData(const Key &key, cpputils::Data data); + static bool _keyHeaderIsCorrect(const Key &key, const cpputils::Data &data); + + std::mutex _mutex; + + DISALLOW_COPY_AND_ASSIGN(EncryptedBlock); +}; + +template +constexpr unsigned int EncryptedBlock::HEADER_LENGTH; + + +template +boost::optional>> EncryptedBlock::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey) { + cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, std::move(data)); + cpputils::Data encrypted = Cipher::encrypt((byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), encKey); + auto baseBlock = baseBlockStore->tryCreate(key, std::move(encrypted)); + if (baseBlock == boost::none) { + //TODO Test this code branch + return boost::none; + } + + return cpputils::make_unique_ref(std::move(*baseBlock), encKey, std::move(plaintextWithHeader)); +} + +template +boost::optional>> EncryptedBlock::TryDecrypt(cpputils::unique_ref baseBlock, const typename Cipher::EncryptionKey &encKey) { + //TODO Change BlockStore so we can read their "class Data" objects instead of "void *data()", and then we can change the Cipher interface to take Data objects instead of "byte *" + size + boost::optional plaintextWithHeader = Cipher::decrypt((byte*)baseBlock->data(), baseBlock->size(), encKey); + if(plaintextWithHeader == boost::none) { + //Decryption failed (e.g. an authenticated cipher detected modifications to the ciphertext) + cpputils::logging::LOG(cpputils::logging::WARN) << "Decrypting block " << baseBlock->key().ToString() << " failed. Was the block modified by an attacker?"; + return boost::none; + } + if(!_keyHeaderIsCorrect(baseBlock->key(), *plaintextWithHeader)) { + //The stored key in the block data is incorrect - an attacker might have exchanged the contents with the encrypted data from a different block + cpputils::logging::LOG(cpputils::logging::WARN) << "Decrypting block " << baseBlock->key().ToString() << " failed due to invalid block key. Was the block modified by an attacker?"; + return boost::none; + } + return cpputils::make_unique_ref>(std::move(baseBlock), encKey, std::move(*plaintextWithHeader)); +} + +template +cpputils::Data EncryptedBlock::_prependKeyHeaderToData(const Key &key, cpputils::Data data) { + static_assert(HEADER_LENGTH >= Key::BINARY_LENGTH, "Key doesn't fit into the header"); + cpputils::Data result(data.size() + HEADER_LENGTH); + std::memcpy(result.data(), key.data(), Key::BINARY_LENGTH); + std::memcpy((uint8_t*)result.data() + Key::BINARY_LENGTH, data.data(), data.size()); + return result; +} + +template +bool EncryptedBlock::_keyHeaderIsCorrect(const Key &key, const cpputils::Data &data) { + return 0 == std::memcmp(key.data(), data.data(), Key::BINARY_LENGTH); +} + +template +EncryptedBlock::EncryptedBlock(cpputils::unique_ref baseBlock, const typename Cipher::EncryptionKey &encKey, cpputils::Data plaintextWithHeader) + :Block(baseBlock->key()), + _baseBlock(std::move(baseBlock)), + _plaintextWithHeader(std::move(plaintextWithHeader)), + _encKey(encKey), + _dataChanged(false), + _mutex() { +} + +template +EncryptedBlock::~EncryptedBlock() { + std::unique_lock lock(_mutex); + _encryptToBaseBlock(); +} + +template +const void *EncryptedBlock::data() const { + return (uint8_t*)_plaintextWithHeader.data() + HEADER_LENGTH; +} + +template +void EncryptedBlock::write(const void *source, uint64_t offset, uint64_t count) { + ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition + std::memcpy((uint8_t*)_plaintextWithHeader.data()+HEADER_LENGTH+offset, source, count); + _dataChanged = true; +} + +template +void EncryptedBlock::flush() { + std::unique_lock lock(_mutex); + _encryptToBaseBlock(); + return _baseBlock->flush(); +} + +template +size_t EncryptedBlock::size() const { + return _plaintextWithHeader.size() - HEADER_LENGTH; +} + +template +void EncryptedBlock::resize(size_t newSize) { + _plaintextWithHeader = cpputils::DataUtils::resize(std::move(_plaintextWithHeader), newSize + HEADER_LENGTH); + _dataChanged = true; +} + +template +void EncryptedBlock::_encryptToBaseBlock() { + if (_dataChanged) { + cpputils::Data encrypted = Cipher::encrypt((byte*)_plaintextWithHeader.data(), _plaintextWithHeader.size(), _encKey); + _baseBlock->write(encrypted.data(), 0, encrypted.size()); + _dataChanged = false; + } +} + +template +cpputils::unique_ref EncryptedBlock::releaseBlock() { + std::unique_lock lock(_mutex); + _encryptToBaseBlock(); + return std::move(_baseBlock); +} + +} +} + +#endif diff --git a/src/blockstore/implementations/encrypted/EncryptedBlockStore.cpp b/src/blockstore/implementations/encrypted/EncryptedBlockStore.cpp new file mode 100644 index 00000000..5ab7ca32 --- /dev/null +++ b/src/blockstore/implementations/encrypted/EncryptedBlockStore.cpp @@ -0,0 +1 @@ +#include "EncryptedBlockStore.h" diff --git a/src/blockstore/implementations/encrypted/EncryptedBlockStore.h b/src/blockstore/implementations/encrypted/EncryptedBlockStore.h new file mode 100644 index 00000000..2f65be01 --- /dev/null +++ b/src/blockstore/implementations/encrypted/EncryptedBlockStore.h @@ -0,0 +1,90 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_ + +#include "../../interface/BlockStore.h" +#include +#include +#include "EncryptedBlock.h" +#include + +namespace blockstore { +namespace encrypted { + +template +class EncryptedBlockStore final: public BlockStore { +public: + EncryptedBlockStore(cpputils::unique_ref baseBlockStore, const typename Cipher::EncryptionKey &encKey); + + //TODO Are createKey() tests included in generic BlockStoreTest? If not, add it! + Key createKey() override; + boost::optional> tryCreate(const Key &key, cpputils::Data data) override; + boost::optional> load(const Key &key) override; + void remove(cpputils::unique_ref block) override; + uint64_t numBlocks() const override; + + //This function should only be used by test cases + void __setKey(const typename Cipher::EncryptionKey &encKey); + +private: + cpputils::unique_ref _baseBlockStore; + typename Cipher::EncryptionKey _encKey; + + DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore); +}; + + + +template +EncryptedBlockStore::EncryptedBlockStore(cpputils::unique_ref baseBlockStore, const typename Cipher::EncryptionKey &encKey) + : _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) { +} + +template +Key EncryptedBlockStore::createKey() { + return _baseBlockStore->createKey(); +} + +template +boost::optional> EncryptedBlockStore::tryCreate(const Key &key, cpputils::Data data) { + //TODO Test that this returns boost::none when base blockstore returns nullptr (for all pass-through-blockstores) + //TODO Easier implementation? This is only so complicated because of the case EncryptedBlock -> Block + auto result = EncryptedBlock::TryCreateNew(_baseBlockStore.get(), key, std::move(data), _encKey); + if (result == boost::none) { + return boost::none; + } + return cpputils::unique_ref(std::move(*result)); +} + +template +boost::optional> EncryptedBlockStore::load(const Key &key) { + auto block = _baseBlockStore->load(key); + if (block == boost::none) { + //TODO Test this path (for all pass-through-blockstores) + return boost::none; + } + return boost::optional>(EncryptedBlock::TryDecrypt(std::move(*block), _encKey)); +} + +template +void EncryptedBlockStore::remove(cpputils::unique_ref block) { + auto encryptedBlock = cpputils::dynamic_pointer_move>(block); + ASSERT(encryptedBlock != boost::none, "Block is not an EncryptedBlock"); + auto baseBlock = (*encryptedBlock)->releaseBlock(); + return _baseBlockStore->remove(std::move(baseBlock)); +} + +template +uint64_t EncryptedBlockStore::numBlocks() const { + return _baseBlockStore->numBlocks(); +} + +template +void EncryptedBlockStore::__setKey(const typename Cipher::EncryptionKey &encKey) { + _encKey = encKey; +} + +} +} + +#endif diff --git a/src/blockstore/implementations/inmemory/InMemoryBlock.cpp b/src/blockstore/implementations/inmemory/InMemoryBlock.cpp new file mode 100644 index 00000000..48bd394f --- /dev/null +++ b/src/blockstore/implementations/inmemory/InMemoryBlock.cpp @@ -0,0 +1,50 @@ +#include "InMemoryBlock.h" +#include "InMemoryBlockStore.h" +#include +#include +#include + +using std::make_shared; +using std::istream; +using std::ostream; +using std::ifstream; +using std::ofstream; +using std::ios; +using cpputils::Data; + +namespace blockstore { +namespace inmemory { + +InMemoryBlock::InMemoryBlock(const Key &key, Data data) + : Block(key), _data(make_shared(std::move(data))) { +} + +InMemoryBlock::InMemoryBlock(const InMemoryBlock &rhs) + : Block(rhs), _data(rhs._data) { +} + +InMemoryBlock::~InMemoryBlock() { +} + +const void *InMemoryBlock::data() const { + return _data->data(); +} + +void InMemoryBlock::write(const void *source, uint64_t offset, uint64_t size) { + ASSERT(offset <= _data->size() && offset + size <= _data->size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition + std::memcpy((uint8_t*)_data->data()+offset, source, size); +} + +size_t InMemoryBlock::size() const { + return _data->size(); +} + +void InMemoryBlock::resize(size_t newSize) { + *_data = cpputils::DataUtils::resize(std::move(*_data), newSize); +} + +void InMemoryBlock::flush() { +} + +} +} diff --git a/src/blockstore/implementations/inmemory/InMemoryBlock.h b/src/blockstore/implementations/inmemory/InMemoryBlock.h new file mode 100644 index 00000000..f85af9a8 --- /dev/null +++ b/src/blockstore/implementations/inmemory/InMemoryBlock.h @@ -0,0 +1,33 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_ + +#include "../../interface/Block.h" +#include + +namespace blockstore { +namespace inmemory { +class InMemoryBlockStore; + +class InMemoryBlock final: public Block { +public: + InMemoryBlock(const Key &key, cpputils::Data size); + InMemoryBlock(const InMemoryBlock &rhs); + ~InMemoryBlock(); + + const void *data() const override; + void write(const void *source, uint64_t offset, uint64_t size) override; + + void flush() override; + + size_t size() const override; + void resize(size_t newSize) override; + +private: + std::shared_ptr _data; +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp b/src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp new file mode 100644 index 00000000..6a97dd7b --- /dev/null +++ b/src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp @@ -0,0 +1,56 @@ +#include "InMemoryBlock.h" +#include "InMemoryBlockStore.h" +#include +#include + +using std::make_unique; +using std::string; +using std::mutex; +using std::lock_guard; +using std::piecewise_construct; +using std::make_tuple; +using cpputils::Data; +using cpputils::unique_ref; +using cpputils::make_unique_ref; +using boost::optional; +using boost::none; + +namespace blockstore { +namespace inmemory { + +InMemoryBlockStore::InMemoryBlockStore() + : _blocks() {} + +optional> InMemoryBlockStore::tryCreate(const Key &key, Data data) { + auto insert_result = _blocks.emplace(piecewise_construct, make_tuple(key.ToString()), make_tuple(key, std::move(data))); + + if (!insert_result.second) { + return none; + } + + //Return a pointer to the stored InMemoryBlock + return optional>(make_unique_ref(insert_result.first->second)); +} + +optional> InMemoryBlockStore::load(const Key &key) { + //Return a pointer to the stored InMemoryBlock + try { + return optional>(make_unique_ref(_blocks.at(key.ToString()))); + } catch (const std::out_of_range &e) { + return none; + } +} + +void InMemoryBlockStore::remove(unique_ref block) { + Key key = block->key(); + cpputils::destruct(std::move(block)); + int numRemoved = _blocks.erase(key.ToString()); + ASSERT(1==numRemoved, "Didn't find block to remove"); +} + +uint64_t InMemoryBlockStore::numBlocks() const { + return _blocks.size(); +} + +} +} diff --git a/src/blockstore/implementations/inmemory/InMemoryBlockStore.h b/src/blockstore/implementations/inmemory/InMemoryBlockStore.h new file mode 100644 index 00000000..2fcb5861 --- /dev/null +++ b/src/blockstore/implementations/inmemory/InMemoryBlockStore.h @@ -0,0 +1,33 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_ + +#include "../../interface/helpers/BlockStoreWithRandomKeys.h" +#include + +#include +#include + +namespace blockstore { +namespace inmemory { +class InMemoryBlock; + +class InMemoryBlockStore final: public BlockStoreWithRandomKeys { +public: + InMemoryBlockStore(); + + boost::optional> tryCreate(const Key &key, cpputils::Data data) override; + boost::optional> load(const Key &key) override; + void remove(cpputils::unique_ref block) override; + uint64_t numBlocks() const override; + +private: + std::map _blocks; + + DISALLOW_COPY_AND_ASSIGN(InMemoryBlockStore); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/ondisk/OnDiskBlock.cpp b/src/blockstore/implementations/ondisk/OnDiskBlock.cpp new file mode 100644 index 00000000..0458df66 --- /dev/null +++ b/src/blockstore/implementations/ondisk/OnDiskBlock.cpp @@ -0,0 +1,108 @@ +#include +#include +#include +#include "OnDiskBlock.h" +#include "OnDiskBlockStore.h" +#include "../../utils/FileDoesntExistException.h" +#include +#include + +using std::istream; +using std::ostream; +using std::ifstream; +using std::ofstream; +using std::ios; +using cpputils::Data; +using cpputils::make_unique_ref; +using cpputils::unique_ref; +using boost::optional; +using boost::none; + +namespace bf = boost::filesystem; + +namespace blockstore { +namespace ondisk { + +OnDiskBlock::OnDiskBlock(const Key &key, const bf::path &filepath, Data data) + : Block(key), _filepath(filepath), _data(std::move(data)), _dataChanged(false), _mutex() { +} + +OnDiskBlock::~OnDiskBlock() { + flush(); +} + +const void *OnDiskBlock::data() const { + return _data.data(); +} + +void OnDiskBlock::write(const void *source, uint64_t offset, uint64_t size) { + ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition + std::memcpy((uint8_t*)_data.data()+offset, source, size); + _dataChanged = true; +} + +size_t OnDiskBlock::size() const { + return _data.size(); +} + +void OnDiskBlock::resize(size_t newSize) { + _data = cpputils::DataUtils::resize(std::move(_data), newSize); + _dataChanged = true; +} + +optional> OnDiskBlock::LoadFromDisk(const bf::path &rootdir, const Key &key) { + auto filepath = rootdir / key.ToString(); + try { + //If it isn't a file, Data::LoadFromFile() would usually also crash. We still need this extra check + //upfront, because Data::LoadFromFile() doesn't crash if we give it the path of a directory + //instead the path of a file. + //TODO Data::LoadFromFile now returns boost::optional. Do we then still need this? + if(!bf::is_regular_file(filepath)) { + return none; + } + boost::optional data = Data::LoadFromFile(filepath); + if (!data) { + return none; + } + return make_unique_ref(key, filepath, std::move(*data)); + } catch (const FileDoesntExistException &e) { + return none; + } +} + +optional> OnDiskBlock::CreateOnDisk(const bf::path &rootdir, const Key &key, Data data) { + auto filepath = rootdir / key.ToString(); + if (bf::exists(filepath)) { + return none; + } + + auto block = make_unique_ref(key, filepath, std::move(data)); + block->_storeToDisk(); + return std::move(block); +} + +void OnDiskBlock::RemoveFromDisk(const bf::path &rootdir, const Key &key) { + auto filepath = rootdir / key.ToString(); + ASSERT(bf::is_regular_file(filepath), "Block not found on disk"); + bf::remove(filepath); +} + +void OnDiskBlock::_fillDataWithZeroes() { + _data.FillWithZeroes(); + _dataChanged = true; +} + +void OnDiskBlock::_storeToDisk() const { + _data.StoreToFile(_filepath); +} + +void OnDiskBlock::flush() { + std::unique_lock lock(_mutex); + if (_dataChanged) { + _storeToDisk(); + _dataChanged = false; + } +} + +} +} diff --git a/src/blockstore/implementations/ondisk/OnDiskBlock.h b/src/blockstore/implementations/ondisk/OnDiskBlock.h new file mode 100644 index 00000000..f17e4d33 --- /dev/null +++ b/src/blockstore/implementations/ondisk/OnDiskBlock.h @@ -0,0 +1,50 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_ + +#include +#include "../../interface/Block.h" +#include +#include + +#include +#include + +namespace blockstore { +namespace ondisk { +class OnDiskBlockStore; + +class OnDiskBlock final: public Block { +public: + OnDiskBlock(const Key &key, const boost::filesystem::path &filepath, cpputils::Data data); + ~OnDiskBlock(); + + static boost::optional> LoadFromDisk(const boost::filesystem::path &rootdir, const Key &key); + static boost::optional> CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data); + static void RemoveFromDisk(const boost::filesystem::path &rootdir, const Key &key); + + const void *data() const override; + void write(const void *source, uint64_t offset, uint64_t size) override; + + void flush() override; + + size_t size() const override; + void resize(size_t newSize) override; + +private: + const boost::filesystem::path _filepath; + cpputils::Data _data; + bool _dataChanged; + + void _fillDataWithZeroes(); + void _storeToDisk() const; + + std::mutex _mutex; + + DISALLOW_COPY_AND_ASSIGN(OnDiskBlock); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp b/src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp new file mode 100644 index 00000000..9ef83446 --- /dev/null +++ b/src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp @@ -0,0 +1,52 @@ +#include "OnDiskBlock.h" +#include "OnDiskBlockStore.h" + +using std::string; +using cpputils::Data; +using cpputils::unique_ref; +using boost::optional; +using boost::none; + +namespace bf = boost::filesystem; + +namespace blockstore { +namespace ondisk { + +OnDiskBlockStore::OnDiskBlockStore(const boost::filesystem::path &rootdir) + : _rootdir(rootdir) { + if (!bf::exists(rootdir)) { + throw std::runtime_error("Base directory not found"); + } + if (!bf::is_directory(rootdir)) { + throw std::runtime_error("Base directory is not a directory"); + } + //TODO Test for read access, write access, enter (x) access, and throw runtime_error in case +} + +//TODO Do I have to lock tryCreate/remove and/or load? Or does ParallelAccessBlockStore take care of that? + +optional> OnDiskBlockStore::tryCreate(const Key &key, Data data) { + //TODO Easier implementation? This is only so complicated because of the cast OnDiskBlock -> Block + auto result = std::move(OnDiskBlock::CreateOnDisk(_rootdir, key, std::move(data))); + if (result == boost::none) { + return boost::none; + } + return unique_ref(std::move(*result)); +} + +optional> OnDiskBlockStore::load(const Key &key) { + return optional>(OnDiskBlock::LoadFromDisk(_rootdir, key)); +} + +void OnDiskBlockStore::remove(unique_ref block) { + Key key = block->key(); + cpputils::destruct(std::move(block)); + OnDiskBlock::RemoveFromDisk(_rootdir, key); +} + +uint64_t OnDiskBlockStore::numBlocks() const { + return std::distance(bf::directory_iterator(_rootdir), bf::directory_iterator()); +} + +} +} diff --git a/src/blockstore/implementations/ondisk/OnDiskBlockStore.h b/src/blockstore/implementations/ondisk/OnDiskBlockStore.h new file mode 100644 index 00000000..0de6df83 --- /dev/null +++ b/src/blockstore/implementations/ondisk/OnDiskBlockStore.h @@ -0,0 +1,31 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_ + +#include +#include "../../interface/helpers/BlockStoreWithRandomKeys.h" + +#include "messmer/cpp-utils/macros.h" + +namespace blockstore { +namespace ondisk { + +class OnDiskBlockStore final: public BlockStoreWithRandomKeys { +public: + explicit OnDiskBlockStore(const boost::filesystem::path &rootdir); + + boost::optional> tryCreate(const Key &key, cpputils::Data data) override; + boost::optional> load(const Key &key) override; + void remove(cpputils::unique_ref block) override; + uint64_t numBlocks() const override; + +private: + const boost::filesystem::path _rootdir; + + DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/parallelaccess/BlockRef.cpp b/src/blockstore/implementations/parallelaccess/BlockRef.cpp new file mode 100644 index 00000000..f35230b4 --- /dev/null +++ b/src/blockstore/implementations/parallelaccess/BlockRef.cpp @@ -0,0 +1 @@ +#include "BlockRef.h" diff --git a/src/blockstore/implementations/parallelaccess/BlockRef.h b/src/blockstore/implementations/parallelaccess/BlockRef.h new file mode 100644 index 00000000..0aa070e5 --- /dev/null +++ b/src/blockstore/implementations/parallelaccess/BlockRef.h @@ -0,0 +1,48 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_BLOCKREF_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_BLOCKREF_H_ + +#include +#include "../../interface/Block.h" +#include "messmer/cpp-utils/macros.h" +#include + +namespace blockstore { +namespace parallelaccess { +class ParallelAccessBlockStore; + +class BlockRef final: public Block, public parallelaccessstore::ParallelAccessStore::ResourceRefBase { +public: + //TODO Unneccessarily storing Key twice here (in parent class and in _baseBlock). + explicit BlockRef(Block *baseBlock): Block(baseBlock->key()), _baseBlock(baseBlock) {} + + const void *data() const override { + return _baseBlock->data(); + } + + void write(const void *source, uint64_t offset, uint64_t size) override { + return _baseBlock->write(source, offset, size); + } + + void flush() override { + return _baseBlock->flush(); + } + + size_t size() const override { + return _baseBlock->size(); + } + + void resize(size_t newSize) override { + return _baseBlock->resize(newSize); + } + +private: + Block *_baseBlock; + + DISALLOW_COPY_AND_ASSIGN(BlockRef); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.cpp b/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.cpp new file mode 100644 index 00000000..23c20ea8 --- /dev/null +++ b/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.cpp @@ -0,0 +1,60 @@ +#include "BlockRef.h" +#include "ParallelAccessBlockStore.h" +#include "ParallelAccessBlockStoreAdapter.h" +#include +#include +#include + +using std::string; +using std::promise; +using cpputils::dynamic_pointer_move; +using cpputils::make_unique_ref; +using boost::none; +using cpputils::unique_ref; +using cpputils::make_unique_ref; +using boost::optional; +using boost::none; + +namespace blockstore { +namespace parallelaccess { + +ParallelAccessBlockStore::ParallelAccessBlockStore(unique_ref baseBlockStore) + : _baseBlockStore(std::move(baseBlockStore)), _parallelAccessStore(make_unique_ref(_baseBlockStore.get())) { +} + +Key ParallelAccessBlockStore::createKey() { + return _baseBlockStore->createKey(); +} + +optional> ParallelAccessBlockStore::tryCreate(const Key &key, cpputils::Data data) { + ASSERT(!_parallelAccessStore.isOpened(key), ("Key "+key.ToString()+"already exists").c_str()); + auto block = _baseBlockStore->tryCreate(key, std::move(data)); + if (block == none) { + //TODO Test this code branch + return none; + } + return unique_ref(_parallelAccessStore.add(key, std::move(*block))); +} + +optional> ParallelAccessBlockStore::load(const Key &key) { + auto block = _parallelAccessStore.load(key); + if (block == none) { + return none; + } + return unique_ref(std::move(*block)); +} + + +void ParallelAccessBlockStore::remove(unique_ref block) { + Key key = block->key(); + auto block_ref = dynamic_pointer_move(block); + ASSERT(block_ref != none, "Block is not a BlockRef"); + return _parallelAccessStore.remove(key, std::move(*block_ref)); +} + +uint64_t ParallelAccessBlockStore::numBlocks() const { + return _baseBlockStore->numBlocks(); +} + +} +} diff --git a/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h b/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h new file mode 100644 index 00000000..ad1c03a0 --- /dev/null +++ b/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h @@ -0,0 +1,34 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTORE_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTORE_H_ + +#include +#include "BlockRef.h" +#include "../../interface/BlockStore.h" +#include + +namespace blockstore { +namespace parallelaccess { + +//TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel) +class ParallelAccessBlockStore final: public BlockStore { +public: + explicit ParallelAccessBlockStore(cpputils::unique_ref baseBlockStore); + + Key createKey() override; + boost::optional> tryCreate(const Key &key, cpputils::Data data) override; + boost::optional> load(const Key &key) override; + void remove(cpputils::unique_ref block) override; + uint64_t numBlocks() const override; + +private: + cpputils::unique_ref _baseBlockStore; + parallelaccessstore::ParallelAccessStore _parallelAccessStore; + + DISALLOW_COPY_AND_ASSIGN(ParallelAccessBlockStore); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp b/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp new file mode 100644 index 00000000..49b896b0 --- /dev/null +++ b/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp @@ -0,0 +1 @@ +#include "ParallelAccessBlockStoreAdapter.h" diff --git a/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.h b/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.h new file mode 100644 index 00000000..ed11256e --- /dev/null +++ b/src/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.h @@ -0,0 +1,35 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTOREADAPTER_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_PARALLELACCESS_PARALLELACCESSBLOCKSTOREADAPTER_H_ + +#include +#include +#include "../../interface/BlockStore.h" + +namespace blockstore { +namespace parallelaccess { + +class ParallelAccessBlockStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore { +public: + explicit ParallelAccessBlockStoreAdapter(BlockStore *baseBlockStore) + :_baseBlockStore(std::move(baseBlockStore)) { + } + + boost::optional> loadFromBaseStore(const Key &key) override { + return _baseBlockStore->load(key); + } + + void removeFromBaseStore(cpputils::unique_ref block) override { + return _baseBlockStore->remove(std::move(block)); + } + +private: + BlockStore *_baseBlockStore; + + DISALLOW_COPY_AND_ASSIGN(ParallelAccessBlockStoreAdapter); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/testfake/FakeBlock.cpp b/src/blockstore/implementations/testfake/FakeBlock.cpp new file mode 100644 index 00000000..098c0ca1 --- /dev/null +++ b/src/blockstore/implementations/testfake/FakeBlock.cpp @@ -0,0 +1,54 @@ +#include "FakeBlock.h" +#include "FakeBlockStore.h" +#include +#include +#include + +using std::shared_ptr; +using std::istream; +using std::ostream; +using std::ifstream; +using std::ofstream; +using std::ios; +using std::string; +using cpputils::Data; + +namespace blockstore { +namespace testfake { + +FakeBlock::FakeBlock(FakeBlockStore *store, const Key &key, shared_ptr data, bool dirty) + : Block(key), _store(store), _data(data), _dataChanged(dirty) { +} + +FakeBlock::~FakeBlock() { + flush(); +} + +const void *FakeBlock::data() const { + return _data->data(); +} + +void FakeBlock::write(const void *source, uint64_t offset, uint64_t size) { + ASSERT(offset <= _data->size() && offset + size <= _data->size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition + std::memcpy((uint8_t*)_data->data()+offset, source, size); + _dataChanged = true; +} + +size_t FakeBlock::size() const { + return _data->size(); +} + +void FakeBlock::resize(size_t newSize) { + *_data = cpputils::DataUtils::resize(std::move(*_data), newSize); + _dataChanged = true; +} + +void FakeBlock::flush() { + if(_dataChanged) { + _store->updateData(key(), *_data); + _dataChanged = false; + } +} + +} +} diff --git a/src/blockstore/implementations/testfake/FakeBlock.h b/src/blockstore/implementations/testfake/FakeBlock.h new file mode 100644 index 00000000..325d5bf7 --- /dev/null +++ b/src/blockstore/implementations/testfake/FakeBlock.h @@ -0,0 +1,39 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCK_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCK_H_ + +#include "../../interface/Block.h" +#include + +#include "messmer/cpp-utils/macros.h" + +namespace blockstore { +namespace testfake { +class FakeBlockStore; + +class FakeBlock final: public Block { +public: + FakeBlock(FakeBlockStore *store, const Key &key, std::shared_ptr data, bool dirty); + ~FakeBlock(); + + const void *data() const override; + void write(const void *source, uint64_t offset, uint64_t size) override; + + void flush() override; + + size_t size() const override; + + void resize(size_t newSize) override; + +private: + FakeBlockStore *_store; + std::shared_ptr _data; + bool _dataChanged; + + DISALLOW_COPY_AND_ASSIGN(FakeBlock); +}; + +} +} + +#endif diff --git a/src/blockstore/implementations/testfake/FakeBlockStore.cpp b/src/blockstore/implementations/testfake/FakeBlockStore.cpp new file mode 100644 index 00000000..1b85a760 --- /dev/null +++ b/src/blockstore/implementations/testfake/FakeBlockStore.cpp @@ -0,0 +1,80 @@ +#include "FakeBlock.h" +#include "FakeBlockStore.h" +#include + +using std::make_shared; +using std::string; +using std::mutex; +using std::lock_guard; +using cpputils::Data; +using cpputils::unique_ref; +using cpputils::make_unique_ref; +using boost::optional; +using boost::none; + +namespace blockstore { +namespace testfake { + +FakeBlockStore::FakeBlockStore() + : _blocks(), _used_dataregions_for_blocks(), _mutex() {} + +optional> FakeBlockStore::tryCreate(const Key &key, Data data) { + std::unique_lock lock(_mutex); + auto insert_result = _blocks.emplace(key.ToString(), std::move(data)); + + if (!insert_result.second) { + return none; + } + + //Return a copy of the stored data + return _load(key); +} + +optional> FakeBlockStore::load(const Key &key) { + std::unique_lock lock(_mutex); + return _load(key); +} + +optional> FakeBlockStore::_load(const Key &key) { + //Return a copy of the stored data + string key_string = key.ToString(); + try { + return makeFakeBlockFromData(key, _blocks.at(key_string), false); + } catch (const std::out_of_range &e) { + return none; + } +} + +void FakeBlockStore::remove(unique_ref block) { + Key key = block->key(); + cpputils::destruct(std::move(block)); + std::unique_lock lock(_mutex); + int numRemoved = _blocks.erase(key.ToString()); + ASSERT(numRemoved == 1, "Block not found"); +} + +unique_ref FakeBlockStore::makeFakeBlockFromData(const Key &key, const Data &data, bool dirty) { + auto newdata = make_shared(data.copy()); + _used_dataregions_for_blocks.push_back(newdata); + return make_unique_ref(this, key, newdata, dirty); +} + +void FakeBlockStore::updateData(const Key &key, const Data &data) { + std::unique_lock lock(_mutex); + auto found = _blocks.find(key.ToString()); + if (found == _blocks.end()) { + auto insertResult = _blocks.emplace(key.ToString(), data.copy()); + ASSERT(true == insertResult.second, "Inserting didn't work"); + found = insertResult.first; + } + Data &stored_data = found->second; + stored_data = data.copy(); +} + +uint64_t FakeBlockStore::numBlocks() const { + std::unique_lock lock(_mutex); + return _blocks.size(); +} + +} +} diff --git a/src/blockstore/implementations/testfake/FakeBlockStore.h b/src/blockstore/implementations/testfake/FakeBlockStore.h new file mode 100644 index 00000000..5fd80502 --- /dev/null +++ b/src/blockstore/implementations/testfake/FakeBlockStore.h @@ -0,0 +1,62 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCKSTORE_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_TESTFAKE_FAKEBLOCKSTORE_H_ + +#include "../../interface/helpers/BlockStoreWithRandomKeys.h" +#include +#include "messmer/cpp-utils/macros.h" + +#include +#include + +namespace blockstore { +namespace testfake { +class FakeBlock; + +/** + * This blockstore is meant to be used for unit tests when the module under test needs a blockstore to work with. + * It basically is the same as the InMemoryBlockStore, but much less forgiving for programming mistakes. + * + * InMemoryBlockStore for example simply ignores flushing and gives you access to the same data region each time + * you request a block. This is very performant, but also forgiving to mistakes. Say you write over the boundaries + * of a block, then you wouldn't notice, since the next time you access the block, the overflow data is (probably) + * still there. Or say an application is relying on flushing the block store in the right moment. Since flushing + * is a no-op in InMemoryBlockStore, you wouldn't notice either. + * + * So this FakeBlockStore has a background copy of each block. When you request a block, you will get a copy of + * the data (instead of a direct pointer as InMemoryBlockStore does) and flushing will copy the data back to the + * background. This way, tests are more likely to fail if they use the blockstore wrongly. + */ +class FakeBlockStore final: public BlockStoreWithRandomKeys { +public: + FakeBlockStore(); + + boost::optional> tryCreate(const Key &key, cpputils::Data data) override; + boost::optional> load(const Key &key) override; + void remove(cpputils::unique_ref block) override; + uint64_t numBlocks() const override; + + void updateData(const Key &key, const cpputils::Data &data); + +private: + std::map _blocks; + + //This vector keeps a handle of the data regions for all created FakeBlock objects. + //This way, it is ensured that no two created FakeBlock objects will work on the + //same data region. Without this, it could happen that a test case creates a FakeBlock, + //destructs it, creates another one, and the new one gets the same memory region. + //We want to avoid this for the reasons mentioned above (overflow data). + std::vector> _used_dataregions_for_blocks; + + mutable std::mutex _mutex; + + cpputils::unique_ref makeFakeBlockFromData(const Key &key, const cpputils::Data &data, bool dirty); + boost::optional> _load(const Key &key); + + DISALLOW_COPY_AND_ASSIGN(FakeBlockStore); +}; + +} +} + +#endif diff --git a/src/blockstore/interface/Block.h b/src/blockstore/interface/Block.h new file mode 100644 index 00000000..0261a097 --- /dev/null +++ b/src/blockstore/interface/Block.h @@ -0,0 +1,41 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_INTERFACE_BLOCK_H_ +#define MESSMER_BLOCKSTORE_INTERFACE_BLOCK_H_ + +#include "../utils/Key.h" +#include + +namespace blockstore { + +//TODO Make Block non-virtual class that stores ptr to its blockstore and writes itself back to the blockstore who is offering a corresponding function. +// Then ondisk blockstore can be actually create the file on disk in blockstore::create() and cachingblockstore will delay that call to its base block store. + +class Block { +public: + virtual ~Block() {} + + virtual const void *data() const = 0; + virtual void write(const void *source, uint64_t offset, uint64_t size) = 0; + + virtual void flush() = 0; + + virtual size_t size() const = 0; + + //TODO Test resize() + virtual void resize(size_t newSize) = 0; + + const Key &key() const { + return _key; + } + +protected: + Block(const Key &key) : _key(key) {} + +private: + const Key _key; +}; + +} + + +#endif diff --git a/src/blockstore/interface/BlockStore.h b/src/blockstore/interface/BlockStore.h new file mode 100644 index 00000000..3af2f3ce --- /dev/null +++ b/src/blockstore/interface/BlockStore.h @@ -0,0 +1,39 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_INTERFACE_BLOCKSTORE_H_ +#define MESSMER_BLOCKSTORE_INTERFACE_BLOCKSTORE_H_ + +#include "Block.h" +#include +#include +#include +#include + +namespace blockstore { + +class BlockStore { +public: + virtual ~BlockStore() {} + + virtual Key createKey() = 0; + //Returns boost::none if key already exists + virtual boost::optional> tryCreate(const Key &key, cpputils::Data data) = 0; + //TODO Use boost::optional (if key doesn't exist) + // Return nullptr if block with this key doesn't exists + virtual boost::optional> load(const Key &key) = 0; + virtual void remove(cpputils::unique_ref block) = 0; + virtual uint64_t numBlocks() const = 0; + + cpputils::unique_ref create(const cpputils::Data &data) { + while(true) { + //TODO Copy (data.copy()) necessary? + auto block = tryCreate(createKey(), data.copy()); + if (block != boost::none) { + return std::move(*block); + } + } + } +}; + +} + +#endif diff --git a/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.cpp b/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.cpp new file mode 100644 index 00000000..f9d7c120 --- /dev/null +++ b/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.cpp @@ -0,0 +1 @@ +#include "BlockStoreWithRandomKeys.h" diff --git a/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.h b/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.h new file mode 100644 index 00000000..9230cabd --- /dev/null +++ b/src/blockstore/interface/helpers/BlockStoreWithRandomKeys.h @@ -0,0 +1,23 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_INTERFACE_HELPERS_BLOCKSTOREWITHRANDOMKEYS_H_ +#define MESSMER_BLOCKSTORE_INTERFACE_HELPERS_BLOCKSTOREWITHRANDOMKEYS_H_ + +#include "../BlockStore.h" +#include "../Block.h" +#include + +namespace blockstore { + +// This is an implementation helpers for BlockStores that use random block keys. +// You should never give this static type to the client. The client should always +// work with the BlockStore interface instead. +class BlockStoreWithRandomKeys: public BlockStore { +public: + Key createKey() final { + return cpputils::Random::PseudoRandom().getFixedSize(); + } +}; + +} + +#endif diff --git a/src/blockstore/utils/BlockStoreUtils.cpp b/src/blockstore/utils/BlockStoreUtils.cpp new file mode 100644 index 00000000..a304c91e --- /dev/null +++ b/src/blockstore/utils/BlockStoreUtils.cpp @@ -0,0 +1,31 @@ +#include "../interface/BlockStore.h" +#include "BlockStoreUtils.h" +#include +#include +#include + +using cpputils::Data; +using cpputils::unique_ref; + +namespace blockstore { +namespace utils { + +unique_ref copyToNewBlock(BlockStore *blockStore, const Block &block) { + Data data(block.size()); + std::memcpy(data.data(), block.data(), block.size()); + return blockStore->create(data); +} + +void copyTo(Block *target, const Block &source) { + ASSERT(target->size() == source.size(), "Can't copy block data when blocks have different sizes"); + target->write(source.data(), 0, source.size()); +} + +void fillWithZeroes(Block *target) { + Data zeroes(target->size()); + zeroes.FillWithZeroes(); + target->write(zeroes.data(), 0, target->size()); +} + +} +} diff --git a/src/blockstore/utils/BlockStoreUtils.h b/src/blockstore/utils/BlockStoreUtils.h new file mode 100644 index 00000000..f479ed01 --- /dev/null +++ b/src/blockstore/utils/BlockStoreUtils.h @@ -0,0 +1,19 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_UTILS_BLOCKSTOREUTILS_H_ +#define MESSMER_BLOCKSTORE_UTILS_BLOCKSTOREUTILS_H_ + +#include + +namespace blockstore { +class BlockStore; +class Block; +namespace utils { + +cpputils::unique_ref copyToNewBlock(BlockStore *blockStore, const Block &block); +void copyTo(Block *target, const Block &source); +void fillWithZeroes(Block *target); + +} +} + +#endif diff --git a/src/blockstore/utils/FileDoesntExistException.cpp b/src/blockstore/utils/FileDoesntExistException.cpp new file mode 100644 index 00000000..d8479e6b --- /dev/null +++ b/src/blockstore/utils/FileDoesntExistException.cpp @@ -0,0 +1,17 @@ +#include "FileDoesntExistException.h" + +namespace bf = boost::filesystem; + +using std::runtime_error; +using std::string; + +namespace blockstore { + +FileDoesntExistException::FileDoesntExistException(const bf::path &filepath) +: runtime_error(string("The file ")+filepath.c_str()+" doesn't exist") { +} + +FileDoesntExistException::~FileDoesntExistException() { +} + +} diff --git a/src/blockstore/utils/FileDoesntExistException.h b/src/blockstore/utils/FileDoesntExistException.h new file mode 100644 index 00000000..d03f57ea --- /dev/null +++ b/src/blockstore/utils/FileDoesntExistException.h @@ -0,0 +1,19 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_UTILS_FILEDOESNTEXISTEXCEPTION_H_ +#define MESSMER_BLOCKSTORE_UTILS_FILEDOESNTEXISTEXCEPTION_H_ + +#include + +#include + +namespace blockstore { + +class FileDoesntExistException final: public std::runtime_error { +public: + explicit FileDoesntExistException(const boost::filesystem::path &filepath); + ~FileDoesntExistException(); +}; + +} + +#endif diff --git a/src/blockstore/utils/Key.cpp b/src/blockstore/utils/Key.cpp new file mode 100644 index 00000000..f22451aa --- /dev/null +++ b/src/blockstore/utils/Key.cpp @@ -0,0 +1 @@ +#include "Key.h" diff --git a/src/blockstore/utils/Key.h b/src/blockstore/utils/Key.h new file mode 100644 index 00000000..c395d2ec --- /dev/null +++ b/src/blockstore/utils/Key.h @@ -0,0 +1,32 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_UTILS_KEY_H_ +#define MESSMER_BLOCKSTORE_UTILS_KEY_H_ + +#include +#include + +namespace blockstore { + + // A key here is NOT a key for encryption, but a key as used in key->value mappings ("access handle for a block"). + //TODO Rename to BlockId/BlobId and make it a class containing a FixedSizeData<> member + using Key = cpputils::FixedSizeData<16>; +} + +namespace std { + //Allow using blockstore::Key in std::unordered_map / std::unordered_set + template <> struct hash { + size_t operator()(const blockstore::Key &key) const { + //Keys are random, so it is enough to use the first few bytes as a hash + return *(size_t*)(key.data()); + } + }; + + //Allow using blockstore::Key in std::map / std::set + template <> struct less { + bool operator()(const blockstore::Key &lhs, const blockstore::Key &rhs) const { + return 0 > std::memcmp(lhs.data(), rhs.data(), blockstore::Key::BINARY_LENGTH); + } + }; +} + +#endif diff --git a/test/blockstore/CMakeLists.txt b/test/blockstore/CMakeLists.txt new file mode 100644 index 00000000..db55fdd2 --- /dev/null +++ b/test/blockstore/CMakeLists.txt @@ -0,0 +1,41 @@ +project (blockstore-test) + +set(SOURCES + utils/BlockStoreUtilsTest.cpp + interface/helpers/BlockStoreWithRandomKeysTest.cpp + interface/BlockStoreTest.cpp + interface/BlockTest.cpp + implementations/testfake/TestFakeBlockStoreTest.cpp + implementations/inmemory/InMemoryBlockStoreTest.cpp + implementations/parallelaccess/ParallelAccessBlockStoreTest.cpp + implementations/compressing/CompressingBlockStoreTest.cpp + implementations/compressing/compressors/testutils/CompressorTest.cpp + implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp + implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp + implementations/ondisk/OnDiskBlockStoreTest.cpp + implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp + implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp + implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp + implementations/caching/CachingBlockStoreTest.cpp + implementations/caching/cache/QueueMapTest_Values.cpp + implementations/caching/cache/testutils/MinimalKeyType.cpp + implementations/caching/cache/testutils/CopyableMovableValueType.cpp + implementations/caching/cache/testutils/MinimalValueType.cpp + implementations/caching/cache/testutils/QueueMapTest.cpp + implementations/caching/cache/testutils/CacheTest.cpp + implementations/caching/cache/QueueMapTest_Size.cpp + implementations/caching/cache/CacheTest_MoveConstructor.cpp + implementations/caching/cache/CacheTest_PushAndPop.cpp + implementations/caching/cache/QueueMapTest_MoveConstructor.cpp + implementations/caching/cache/QueueMapTest_MemoryLeak.cpp + implementations/caching/cache/CacheTest_RaceCondition.cpp + implementations/caching/cache/PeriodicTaskTest.cpp + implementations/caching/cache/QueueMapTest_Peek.cpp +) + +add_executable(${PROJECT_NAME} ${SOURCES}) +target_link_libraries(${PROJECT_NAME} googletest blockstore) +add_test(${PROJECT_NAME} ${PROJECT_NAME}) + +target_enable_style_warnings(${PROJECT_NAME}) +target_activate_cpp14(${PROJECT_NAME}) diff --git a/test/blockstore/implementations/caching/CachingBlockStoreTest.cpp b/test/blockstore/implementations/caching/CachingBlockStoreTest.cpp new file mode 100644 index 00000000..312012ae --- /dev/null +++ b/test/blockstore/implementations/caching/CachingBlockStoreTest.cpp @@ -0,0 +1,23 @@ +#include "../../../implementations/caching/CachingBlockStore.h" +#include "../../../implementations/testfake/FakeBlockStore.h" +#include "../../testutils/BlockStoreTest.h" +#include "google/gtest/gtest.h" +#include + + +using blockstore::BlockStore; +using blockstore::caching::CachingBlockStore; +using blockstore::testfake::FakeBlockStore; +using cpputils::unique_ref; +using cpputils::make_unique_ref; + +class CachingBlockStoreTestFixture: public BlockStoreTestFixture { +public: + unique_ref createBlockStore() override { + return make_unique_ref(make_unique_ref()); + } +}; + +INSTANTIATE_TYPED_TEST_CASE_P(Caching, BlockStoreTest, CachingBlockStoreTestFixture); + +//TODO Add specific tests for the blockstore diff --git a/test/blockstore/implementations/caching/cache/CacheTest_MoveConstructor.cpp b/test/blockstore/implementations/caching/cache/CacheTest_MoveConstructor.cpp new file mode 100644 index 00000000..fa00da2d --- /dev/null +++ b/test/blockstore/implementations/caching/cache/CacheTest_MoveConstructor.cpp @@ -0,0 +1,35 @@ +#include +#include +#include "../../../../implementations/caching/cache/Cache.h" +#include "testutils/MinimalKeyType.h" +#include "testutils/CopyableMovableValueType.h" + +using namespace blockstore::caching; + +using cpputils::unique_ref; +using cpputils::make_unique_ref; +using ::testing::Test; + +//Test that Cache uses a move constructor for Value if possible +class CacheTest_MoveConstructor: public Test { +public: + CacheTest_MoveConstructor(): cache(make_unique_ref>()) { + CopyableMovableValueType::numCopyConstructorCalled = 0; + } + unique_ref> cache; +}; + +TEST_F(CacheTest_MoveConstructor, MoveIntoCache) { + cache->push(MinimalKeyType::create(0), CopyableMovableValueType(2)); + CopyableMovableValueType val = cache->pop(MinimalKeyType::create(0)).value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away + EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled); +} + +TEST_F(CacheTest_MoveConstructor, CopyIntoCache) { + CopyableMovableValueType value(2); + cache->push(MinimalKeyType::create(0), value); + CopyableMovableValueType val = cache->pop(MinimalKeyType::create(0)).value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away + EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled); +} diff --git a/test/blockstore/implementations/caching/cache/CacheTest_PushAndPop.cpp b/test/blockstore/implementations/caching/cache/CacheTest_PushAndPop.cpp new file mode 100644 index 00000000..df036f70 --- /dev/null +++ b/test/blockstore/implementations/caching/cache/CacheTest_PushAndPop.cpp @@ -0,0 +1,134 @@ +#include "testutils/CacheTest.h" + +#include "../../../../implementations/caching/cache/Cache.h" +#include "testutils/MinimalKeyType.h" +#include "testutils/MinimalValueType.h" +#include + +using ::testing::Test; + +using namespace blockstore::caching; + +class CacheTest_PushAndPop: public CacheTest {}; + +TEST_F(CacheTest_PushAndPop, PopNonExistingEntry_EmptyCache) { + EXPECT_EQ(boost::none, pop(10)); +} + +TEST_F(CacheTest_PushAndPop, PopNonExistingEntry_NonEmptyCache) { + push(9, 10); + EXPECT_EQ(boost::none, pop(10)); +} + +TEST_F(CacheTest_PushAndPop, PopNonExistingEntry_FullCache) { + //Add a lot of even numbered keys + for (unsigned int i = 0; i < MAX_ENTRIES; ++i) { + push(2*i, 2*i); + } + //Request an odd numbered key + EXPECT_EQ(boost::none, pop(9)); +} + +TEST_F(CacheTest_PushAndPop, OneEntry) { + push(10, 20); + EXPECT_EQ(20, pop(10).value()); +} + +TEST_F(CacheTest_PushAndPop, MultipleEntries) { + push(10, 20); + push(20, 30); + push(30, 40); + EXPECT_EQ(30, pop(20).value()); + EXPECT_EQ(20, pop(10).value()); + EXPECT_EQ(40, pop(30).value()); +} + +TEST_F(CacheTest_PushAndPop, FullCache) { + for(unsigned int i = 0; i < MAX_ENTRIES; ++i) { + push(i, 2*i); + } + for(unsigned int i = 0; i < MAX_ENTRIES; ++i) { + EXPECT_EQ(2*i, pop(i).value()); + } +} + +TEST_F(CacheTest_PushAndPop, FullCache_PushNonOrdered_PopOrdered) { + for(unsigned int i = 1; i < MAX_ENTRIES; i += 2) { + push(i, 2*i); + } + for(unsigned int i = 0; i < MAX_ENTRIES; i += 2) { + push(i, 2*i); + } + for(unsigned int i = 0; i < MAX_ENTRIES; ++i) { + EXPECT_EQ(2*i, pop(i).value()); + } +} + +TEST_F(CacheTest_PushAndPop, FullCache_PushOrdered_PopNonOrdered) { + for(unsigned int i = 0; i < MAX_ENTRIES; ++i) { + push(i, 2*i); + } + for(unsigned int i = 1; i < MAX_ENTRIES; i += 2) { + EXPECT_EQ(2*i, pop(i).value()); + } + for(unsigned int i = 0; i < MAX_ENTRIES; i += 2) { + EXPECT_EQ(2*i, pop(i).value()); + } +} + +int roundDownToEven(int number) { + if (number % 2 == 0) { + return number; + } else { + return number - 1; + } +} + +int roundDownToOdd(int number) { + if (number % 2 != 0) { + return number; + } else { + return number - 1; + } +} + +TEST_F(CacheTest_PushAndPop, FullCache_PushNonOrdered_PopNonOrdered) { + for(int i = roundDownToEven(MAX_ENTRIES - 1); i >= 0; i -= 2) { + push(i, 2*i); + } + for(unsigned int i = 1; i < MAX_ENTRIES; i += 2) { + push(i, 2*i); + } + for(int i = roundDownToOdd(MAX_ENTRIES-1); i >= 0; i -= 2) { + EXPECT_EQ(2*i, pop(i).value()); + } + for(unsigned int i = 0; i < MAX_ENTRIES; i += 2) { + EXPECT_EQ(2*i, pop(i).value()); + } +} + +TEST_F(CacheTest_PushAndPop, MoreThanFullCache) { + for(unsigned int i = 0; i < MAX_ENTRIES + 2; ++i) { + push(i, 2*i); + } + //Check that the oldest two elements got deleted automatically + EXPECT_EQ(boost::none, pop(0)); + EXPECT_EQ(boost::none, pop(1)); + //Check the other elements are still there + for(unsigned int i = 2; i < MAX_ENTRIES + 2; ++i) { + EXPECT_EQ(2*i, pop(i).value()); + } +} + +TEST_F(CacheTest_PushAndPop, AfterTimeout) { + constexpr double TIMEOUT1_SEC = Cache::MAX_LIFETIME_SEC * 3/4; + constexpr double TIMEOUT2_SEC = Cache::PURGE_LIFETIME_SEC * 3/4; + static_assert(TIMEOUT1_SEC + TIMEOUT2_SEC > Cache::MAX_LIFETIME_SEC, "Ensure that our chosen timeouts push the first entry out of the cache"); + + push(10, 20); + boost::this_thread::sleep_for(boost::chrono::milliseconds(static_cast(1000 * TIMEOUT1_SEC))); + push(20, 30); + boost::this_thread::sleep_for(boost::chrono::milliseconds(static_cast(1000 * TIMEOUT2_SEC))); + EXPECT_EQ(boost::none, pop(10)); + EXPECT_EQ(30, pop(20).value()); +} diff --git a/test/blockstore/implementations/caching/cache/CacheTest_RaceCondition.cpp b/test/blockstore/implementations/caching/cache/CacheTest_RaceCondition.cpp new file mode 100644 index 00000000..329f6d22 --- /dev/null +++ b/test/blockstore/implementations/caching/cache/CacheTest_RaceCondition.cpp @@ -0,0 +1,109 @@ +#include "testutils/CacheTest.h" +#include +#include +#include +#include +#include + +using namespace blockstore::caching; +using std::chrono::seconds; +using std::string; +using cpputils::ConditionBarrier; +using std::unique_ptr; +using std::make_unique; +using std::future; + +// Regression tests for a race condition. +// An element could be in the process of being thrown out of the cache and while the destructor is running, another +// thread calls pop() for the element and gets none returned. But since the destructor isn't finished yet, the data from +// the cache element also isn't completely written back yet and an application loading it runs into a race condition. + +class ObjectWithLongDestructor { +public: + ObjectWithLongDestructor(ConditionBarrier *onDestructorStarted, bool *destructorFinished) + : _onDestructorStarted(onDestructorStarted), _destructorFinished(destructorFinished) {} + ~ObjectWithLongDestructor() { + _onDestructorStarted->release(); + std::this_thread::sleep_for(seconds(1)); + *_destructorFinished = true; + } +private: + ConditionBarrier *_onDestructorStarted; + bool *_destructorFinished; + + DISALLOW_COPY_AND_ASSIGN(ObjectWithLongDestructor); +}; + +class CacheTest_RaceCondition: public ::testing::Test { +public: + CacheTest_RaceCondition(): cache(), destructorStarted(), destructorFinished(false) {} + + static constexpr unsigned int MAX_ENTRIES = 100; + + Cache, MAX_ENTRIES> cache; + ConditionBarrier destructorStarted; + bool destructorFinished; + + int pushObjectWithLongDestructor() { + cache.push(2, make_unique(&destructorStarted, &destructorFinished)); + return 2; + } + + int pushDummyObject() { + cache.push(3, nullptr); + return 3; + } + + future causeCacheOverflowInOtherThread() { + //Add maximum+1 element in another thread (this causes the cache to flush the first element in another thread) + return std::async(std::launch::async, [this] { + for(unsigned int i = 0; i < MAX_ENTRIES+1; ++i) { + cache.push(MAX_ENTRIES+i, nullptr); + } + }); + } + + void EXPECT_POP_BLOCKS_UNTIL_DESTRUCTOR_FINISHED(int key) { + EXPECT_FALSE(destructorFinished); + cache.pop(key); + EXPECT_TRUE(destructorFinished); + } + + void EXPECT_POP_DOESNT_BLOCK_UNTIL_DESTRUCTOR_FINISHED(int key) { + EXPECT_FALSE(destructorFinished); + cache.pop(key); + EXPECT_FALSE(destructorFinished); + } +}; + +TEST_F(CacheTest_RaceCondition, PopBlocksWhileRequestedElementIsThrownOut_ByAge) { + auto id = pushObjectWithLongDestructor(); + + destructorStarted.wait(); + EXPECT_POP_BLOCKS_UNTIL_DESTRUCTOR_FINISHED(id); +} + +TEST_F(CacheTest_RaceCondition, PopDoesntBlockWhileOtherElementIsThrownOut_ByAge) { + pushObjectWithLongDestructor(); + auto id = pushDummyObject(); + + destructorStarted.wait(); + EXPECT_POP_DOESNT_BLOCK_UNTIL_DESTRUCTOR_FINISHED(id); +} + +TEST_F(CacheTest_RaceCondition, PopBlocksWhileRequestedElementIsThrownOut_ByPush) { + auto id = pushObjectWithLongDestructor(); + + auto future = causeCacheOverflowInOtherThread(); + destructorStarted.wait(); + EXPECT_POP_BLOCKS_UNTIL_DESTRUCTOR_FINISHED(id); +} + +TEST_F(CacheTest_RaceCondition, PopDoesntBlockWhileOtherElementIsThrownOut_ByPush) { + pushObjectWithLongDestructor(); + auto id = pushDummyObject(); + + auto future = causeCacheOverflowInOtherThread(); + destructorStarted.wait(); + EXPECT_POP_DOESNT_BLOCK_UNTIL_DESTRUCTOR_FINISHED(id); +} diff --git a/test/blockstore/implementations/caching/cache/PeriodicTaskTest.cpp b/test/blockstore/implementations/caching/cache/PeriodicTaskTest.cpp new file mode 100644 index 00000000..d49492ad --- /dev/null +++ b/test/blockstore/implementations/caching/cache/PeriodicTaskTest.cpp @@ -0,0 +1,63 @@ +#include + +#include "../../../../implementations/caching/cache/PeriodicTask.h" + +#include +#include +#include + +using ::testing::Test; +using std::mutex; +using std::unique_lock; +using std::condition_variable; + +using namespace blockstore::caching; + +class AtomicCounter { +public: + AtomicCounter(int count): _mutex(), _cv(), _counter(count) {} + + void decrease() { + unique_lock lock(_mutex); + --_counter; + _cv.notify_all(); + } + + void waitForZero() { + unique_lock lock(_mutex); + _cv.wait(lock, [this] () {return _counter <= 0;}); + } +private: + mutex _mutex; + condition_variable _cv; + int _counter; +}; + +class PeriodicTaskTest: public Test { +}; + +TEST_F(PeriodicTaskTest, DoesntDeadlockInDestructorWhenDestructedImmediately) { + PeriodicTask task([](){}, 1); +} + +TEST_F(PeriodicTaskTest, CallsCallbackAtLeast10Times) { + AtomicCounter counter(10); + + PeriodicTask task([&counter](){ + counter.decrease(); + }, 0.001); + + counter.waitForZero(); +} + +TEST_F(PeriodicTaskTest, DoesntCallCallbackAfterDestruction) { + std::atomic callCount(0); + { + PeriodicTask task([&callCount](){ + callCount += 1; + }, 0.001); + } + int callCountDirectlyAfterDestruction = callCount; + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + EXPECT_EQ(callCountDirectlyAfterDestruction, callCount); +} diff --git a/test/blockstore/implementations/caching/cache/QueueMapTest_MemoryLeak.cpp b/test/blockstore/implementations/caching/cache/QueueMapTest_MemoryLeak.cpp new file mode 100644 index 00000000..74012fc0 --- /dev/null +++ b/test/blockstore/implementations/caching/cache/QueueMapTest_MemoryLeak.cpp @@ -0,0 +1,87 @@ +#include "testutils/QueueMapTest.h" + +// Tests that QueueMap calls destructors correctly. +// This is needed, because QueueMap does its own memory management. +class QueueMapTest_MemoryLeak: public QueueMapTest { +public: + void EXPECT_NUM_INSTANCES(int num) { + EXPECT_EQ(num, MinimalKeyType::instances); + EXPECT_EQ(num, MinimalValueType::instances); + } +}; + +TEST_F(QueueMapTest_MemoryLeak, Empty) { + EXPECT_NUM_INSTANCES(0); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOne) { + push(2, 3); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_NUM_INSTANCES(2); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwoAndPoppingOldest) { + push(2, 3); + push(3, 4); + pop(); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwoAndPoppingLast) { + push(2, 3); + push(3, 4); + pop(3); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOne) { + push(2, 3); + pop(); + EXPECT_NUM_INSTANCES(0); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePerKey) { + push(2, 3); + pop(2); + EXPECT_NUM_INSTANCES(0); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePushingOne) { + push(2, 3); + pop(); + push(3, 4); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePerKeyPushingOne) { + push(2, 3); + pop(2); + push(3, 4); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePushingSame) { + push(2, 3); + pop(); + push(2, 3); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePerKeyPushingSame) { + push(2, 3); + pop(2); + push(2, 3); + EXPECT_NUM_INSTANCES(1); +} diff --git a/test/blockstore/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp b/test/blockstore/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp new file mode 100644 index 00000000..aab5eee9 --- /dev/null +++ b/test/blockstore/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp @@ -0,0 +1,50 @@ +#include +#include +#include "../../../../implementations/caching/cache/QueueMap.h" +#include "testutils/MinimalKeyType.h" +#include "testutils/CopyableMovableValueType.h" + +using namespace blockstore::caching; + +using ::testing::Test; +using cpputils::unique_ref; +using cpputils::make_unique_ref; + +//Test that QueueMap uses a move constructor for Value if possible +class QueueMapTest_MoveConstructor: public Test { +public: + QueueMapTest_MoveConstructor(): map(make_unique_ref>()) { + CopyableMovableValueType::numCopyConstructorCalled = 0; + } + unique_ref> map; +}; + +TEST_F(QueueMapTest_MoveConstructor, PushingAndPopping_MoveIntoMap) { + map->push(MinimalKeyType::create(0), CopyableMovableValueType(2)); + CopyableMovableValueType val = map->pop().value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away + EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled); +} + +TEST_F(QueueMapTest_MoveConstructor, PushingAndPoppingPerKey_MoveIntoMap) { + map->push(MinimalKeyType::create(0), CopyableMovableValueType(2)); + CopyableMovableValueType val = map->pop(MinimalKeyType::create(0)).value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away + EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled); +} + +TEST_F(QueueMapTest_MoveConstructor, PushingAndPopping_CopyIntoMap) { + CopyableMovableValueType value(2); + map->push(MinimalKeyType::create(0), value); + CopyableMovableValueType val = map->pop().value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away + EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled); +} + +TEST_F(QueueMapTest_MoveConstructor, PushingAndPoppingPerKey_CopyIntoMap) { + CopyableMovableValueType value(2); + map->push(MinimalKeyType::create(0), value); + CopyableMovableValueType val = map->pop(MinimalKeyType::create(0)).value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away + EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled); +} diff --git a/test/blockstore/implementations/caching/cache/QueueMapTest_Peek.cpp b/test/blockstore/implementations/caching/cache/QueueMapTest_Peek.cpp new file mode 100644 index 00000000..ba3c7ead --- /dev/null +++ b/test/blockstore/implementations/caching/cache/QueueMapTest_Peek.cpp @@ -0,0 +1,34 @@ +#include "testutils/QueueMapTest.h" +#include + +class QueueMapPeekTest: public QueueMapTest {}; + +TEST_F(QueueMapPeekTest, PoppingFromEmpty) { + EXPECT_EQ(boost::none, peek()); +} + +TEST_F(QueueMapPeekTest, PushingOne) { + push(3, 2); + EXPECT_EQ(2, peek().value()); +} + +TEST_F(QueueMapPeekTest, PushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_EQ(3, peek().value()); + EXPECT_EQ(3, peek().value()); + EXPECT_EQ(3, pop().value()); + EXPECT_EQ(4, peek().value()); + EXPECT_EQ(4, peek().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(boost::none, peek()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapPeekTest, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, peek().value()); +} diff --git a/test/blockstore/implementations/caching/cache/QueueMapTest_Size.cpp b/test/blockstore/implementations/caching/cache/QueueMapTest_Size.cpp new file mode 100644 index 00000000..e4d72c9d --- /dev/null +++ b/test/blockstore/implementations/caching/cache/QueueMapTest_Size.cpp @@ -0,0 +1,79 @@ +#include "testutils/QueueMapTest.h" + +class QueueMapTest_Size: public QueueMapTest {}; + +TEST_F(QueueMapTest_Size, Empty) { + EXPECT_EQ(0, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOne) { + push(2, 3); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_EQ(2, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingTwoAndPoppingOldest) { + push(2, 3); + push(3, 4); + pop(); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingTwoAndPoppingLast) { + push(2, 3); + push(3, 4); + pop(3); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOne) { + push(2, 3); + pop(); + EXPECT_EQ(0, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePerKey) { + push(2, 3); + pop(2); + EXPECT_EQ(0, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePushingOne) { + push(2, 3); + pop(); + push(3, 4); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePerKeyPushingOne) { + push(2, 3); + pop(2); + push(3, 4); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePushingSame) { + push(2, 3); + pop(); + push(2, 3); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePerKeyPushingSame) { + push(2, 3); + pop(2); + push(2, 3); + EXPECT_EQ(1, size()); +} diff --git a/test/blockstore/implementations/caching/cache/QueueMapTest_Values.cpp b/test/blockstore/implementations/caching/cache/QueueMapTest_Values.cpp new file mode 100644 index 00000000..9d5cfcbe --- /dev/null +++ b/test/blockstore/implementations/caching/cache/QueueMapTest_Values.cpp @@ -0,0 +1,151 @@ +#include "testutils/QueueMapTest.h" +#include + +class QueueMapTest_Values: public QueueMapTest {}; + +TEST_F(QueueMapTest_Values, PoppingFromEmpty) { + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, PoppingFromEmptyPerKey) { + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapTest_Values, PoppingNonexistingPerKey) { + push(3, 2); + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapTest_Values, PushingOne) { + push(3, 2); + EXPECT_EQ(2, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, PushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_EQ(3, pop().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, AfterPushingTwoAndPoppingLast) { + push(2, 3); + push(3, 4); + pop(3); + EXPECT_EQ(boost::none, pop(3)); + EXPECT_EQ(3, pop(2).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOne) { + push(2, 3); + pop(); + EXPECT_EQ(boost::none, pop()); + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOnePerKey) { + push(2, 3); + pop(2); + EXPECT_EQ(boost::none, pop()); + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOnePushingOne) { + push(2, 3); + pop(); + push(3, 4); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOnePerKeyPushingOne) { + push(2, 3); + pop(2); + push(3, 4); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, PushingSomePoppingMiddlePerKey) { + push(1, 2); + push(2, 3); + push(3, 4); + push(4, 5); + push(5, 6); + EXPECT_EQ(3, pop(2).value()); + EXPECT_EQ(5, pop(4).value()); + EXPECT_EQ(2, pop().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(6, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, PushingSomePoppingFirstPerKey) { + push(1, 2); + push(2, 3); + push(3, 4); + push(4, 5); + push(5, 6); + EXPECT_EQ(2, pop(1).value()); + EXPECT_EQ(3, pop(2).value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(5, pop().value()); + EXPECT_EQ(6, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, PushingSomePoppingLastPerKey) { + push(1, 2); + push(2, 3); + push(3, 4); + push(4, 5); + push(5, 6); + EXPECT_EQ(6, pop(5).value()); + EXPECT_EQ(5, pop(4).value()); + EXPECT_EQ(2, pop().value()); + EXPECT_EQ(3, pop().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +//This test forces the underlying datastructure (std::map or std::unordered_map) to grow and reallocate memory. +//So it tests, that QueueMap still works after reallocating memory. +TEST_F(QueueMapTest_Values, ManyValues) { + //Push 1 million entries + for (int i = 0; i < 1000000; ++i) { + push(i, 2*i); + } + //pop every other one by key + for (int i = 0; i < 1000000; i += 2) { + EXPECT_EQ(2*i, pop(i).value()); + } + //pop the rest in queue order + for (int i = 1; i < 1000000; i += 2) { + EXPECT_EQ(2*i, peek().value()); + EXPECT_EQ(2*i, pop().value()); + } + EXPECT_EQ(0, size()); + EXPECT_EQ(boost::none, pop()); + EXPECT_EQ(boost::none, peek()); +} + +TEST_F(QueueMapTest_Values, PushAlreadyExistingValue) { + push(2, 3); + EXPECT_ANY_THROW( + push(2, 4); + ); +} diff --git a/test/blockstore/implementations/caching/cache/testutils/CacheTest.cpp b/test/blockstore/implementations/caching/cache/testutils/CacheTest.cpp new file mode 100644 index 00000000..b93bbccb --- /dev/null +++ b/test/blockstore/implementations/caching/cache/testutils/CacheTest.cpp @@ -0,0 +1,13 @@ +#include "CacheTest.h" + +void CacheTest::push(int key, int value) { + return _cache.push(MinimalKeyType::create(key), MinimalValueType::create(value)); +} + +boost::optional CacheTest::pop(int key) { + boost::optional entry = _cache.pop(MinimalKeyType::create(key)); + if (!entry) { + return boost::none; + } + return entry->value(); +} diff --git a/test/blockstore/implementations/caching/cache/testutils/CacheTest.h b/test/blockstore/implementations/caching/cache/testutils/CacheTest.h new file mode 100644 index 00000000..f9000b3f --- /dev/null +++ b/test/blockstore/implementations/caching/cache/testutils/CacheTest.h @@ -0,0 +1,29 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_ +#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_ + +#include +#include "../../../../../implementations/caching/cache/Cache.h" +#include "MinimalKeyType.h" +#include "MinimalValueType.h" +#include + +// This class is a parent class for tests on QueueMap. +// It offers functions to work with a QueueMap test object which is built using types having only the minimal type requirements. +// Furthermore, the class checks that there are no memory leaks left after destructing the QueueMap (by counting leftover instances of Keys/Values). +class CacheTest: public ::testing::Test { +public: + CacheTest(): _cache() {} + + void push(int key, int value); + boost::optional pop(int key); + + static constexpr unsigned int MAX_ENTRIES = 100; + + using Cache = blockstore::caching::Cache; + +private: + Cache _cache; +}; + +#endif diff --git a/test/blockstore/implementations/caching/cache/testutils/CopyableMovableValueType.cpp b/test/blockstore/implementations/caching/cache/testutils/CopyableMovableValueType.cpp new file mode 100644 index 00000000..cf23bd88 --- /dev/null +++ b/test/blockstore/implementations/caching/cache/testutils/CopyableMovableValueType.cpp @@ -0,0 +1,3 @@ +#include "CopyableMovableValueType.h" + +int CopyableMovableValueType::numCopyConstructorCalled = 0; diff --git a/test/blockstore/implementations/caching/cache/testutils/CopyableMovableValueType.h b/test/blockstore/implementations/caching/cache/testutils/CopyableMovableValueType.h new file mode 100644 index 00000000..e69c1de6 --- /dev/null +++ b/test/blockstore/implementations/caching/cache/testutils/CopyableMovableValueType.h @@ -0,0 +1,33 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_COPYABLEMOVABLEVALUETYPE_H_ +#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_COPYABLEMOVABLEVALUETYPE_H_ + +class CopyableMovableValueType { +public: + static int numCopyConstructorCalled; + CopyableMovableValueType(int value): _value(value) {} + CopyableMovableValueType(const CopyableMovableValueType &rhs): CopyableMovableValueType(rhs._value) { + ++numCopyConstructorCalled; + } + CopyableMovableValueType &operator=(const CopyableMovableValueType &rhs) { + _value = rhs._value; + ++numCopyConstructorCalled; + return *this; + } + CopyableMovableValueType(CopyableMovableValueType &&rhs): CopyableMovableValueType(rhs._value) { + //Don't increase numCopyConstructorCalled + } + CopyableMovableValueType &operator=(CopyableMovableValueType &&rhs) { + //Don't increase numCopyConstructorCalled + _value = rhs._value; + return *this; + } + int value() const { + return _value; + } +private: + int _value; +}; + + +#endif diff --git a/test/blockstore/implementations/caching/cache/testutils/MinimalKeyType.cpp b/test/blockstore/implementations/caching/cache/testutils/MinimalKeyType.cpp new file mode 100644 index 00000000..f6b327ff --- /dev/null +++ b/test/blockstore/implementations/caching/cache/testutils/MinimalKeyType.cpp @@ -0,0 +1,3 @@ +#include "MinimalKeyType.h" + +int MinimalKeyType::instances = 0; diff --git a/test/blockstore/implementations/caching/cache/testutils/MinimalKeyType.h b/test/blockstore/implementations/caching/cache/testutils/MinimalKeyType.h new file mode 100644 index 00000000..14dfb5be --- /dev/null +++ b/test/blockstore/implementations/caching/cache/testutils/MinimalKeyType.h @@ -0,0 +1,47 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALKEYTYPE_H_ +#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALKEYTYPE_H_ + +#include + +// This is a not-default-constructible Key type +class MinimalKeyType { +public: + static int instances; + + static MinimalKeyType create(int value) { + return MinimalKeyType(value); + } + + MinimalKeyType(const MinimalKeyType &rhs): MinimalKeyType(rhs.value()) { + } + + ~MinimalKeyType() { + --instances; + } + + int value() const { + return _value; + } + +private: + MinimalKeyType(int value): _value(value) { + ++instances; + } + + int _value; +}; + +namespace std { +template <> struct hash { + size_t operator()(const MinimalKeyType &obj) const { + return obj.value(); + } +}; +} + +inline bool operator==(const MinimalKeyType &lhs, const MinimalKeyType &rhs) { + return lhs.value() == rhs.value(); +} + +#endif diff --git a/test/blockstore/implementations/caching/cache/testutils/MinimalValueType.cpp b/test/blockstore/implementations/caching/cache/testutils/MinimalValueType.cpp new file mode 100644 index 00000000..f5cbdbae --- /dev/null +++ b/test/blockstore/implementations/caching/cache/testutils/MinimalValueType.cpp @@ -0,0 +1,3 @@ +#include "MinimalValueType.h" + +int MinimalValueType::instances = 0; diff --git a/test/blockstore/implementations/caching/cache/testutils/MinimalValueType.h b/test/blockstore/implementations/caching/cache/testutils/MinimalValueType.h new file mode 100644 index 00000000..d2be569f --- /dev/null +++ b/test/blockstore/implementations/caching/cache/testutils/MinimalValueType.h @@ -0,0 +1,52 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALVALUETYPE_H_ +#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALVALUETYPE_H_ + +#include +#include +#include + +// This is a not-default-constructible non-copyable but moveable Value type +class MinimalValueType { +public: + static int instances; + + static MinimalValueType create(int value) { + return MinimalValueType(value); + } + + MinimalValueType(MinimalValueType &&rhs): MinimalValueType(rhs.value()) { + rhs._isMoved = true; + } + + MinimalValueType &operator=(MinimalValueType &&rhs) { + _value = rhs.value(); + _isMoved = false; + rhs._isMoved = true; + return *this; + } + + ~MinimalValueType() { + ASSERT(!_isDestructed, "Object was already destructed before"); + --instances; + _isDestructed = true; + } + + int value() const { + ASSERT(!_isMoved && !_isDestructed, "Object is invalid"); + return _value; + } + +private: + MinimalValueType(int value): _value(value), _isMoved(false), _isDestructed(false) { + ++instances; + } + + int _value; + bool _isMoved; + bool _isDestructed; + + DISALLOW_COPY_AND_ASSIGN(MinimalValueType); +}; + +#endif diff --git a/test/blockstore/implementations/caching/cache/testutils/QueueMapTest.cpp b/test/blockstore/implementations/caching/cache/testutils/QueueMapTest.cpp new file mode 100644 index 00000000..76fae6fe --- /dev/null +++ b/test/blockstore/implementations/caching/cache/testutils/QueueMapTest.cpp @@ -0,0 +1,44 @@ +#include "QueueMapTest.h" + +QueueMapTest::QueueMapTest(): _map(cpputils::make_unique_ref>()) { + MinimalKeyType::instances = 0; + MinimalValueType::instances = 0; +} + +QueueMapTest::~QueueMapTest() { + cpputils::destruct(std::move(_map)); + EXPECT_EQ(0, MinimalKeyType::instances); + EXPECT_EQ(0, MinimalValueType::instances); +} + +void QueueMapTest::push(int key, int value) { + _map->push(MinimalKeyType::create(key), MinimalValueType::create(value)); +} + +boost::optional QueueMapTest::pop() { + auto elem = _map->pop(); + if (!elem) { + return boost::none; + } + return elem.value().value(); +} + +boost::optional QueueMapTest::pop(int key) { + auto elem = _map->pop(MinimalKeyType::create(key)); + if (!elem) { + return boost::none; + } + return elem.value().value(); +} + +boost::optional QueueMapTest::peek() { + auto elem = _map->peek(); + if (!elem) { + return boost::none; + } + return elem.value().value(); +} + +int QueueMapTest::size() { + return _map->size(); +} diff --git a/test/blockstore/implementations/caching/cache/testutils/QueueMapTest.h b/test/blockstore/implementations/caching/cache/testutils/QueueMapTest.h new file mode 100644 index 00000000..59796803 --- /dev/null +++ b/test/blockstore/implementations/caching/cache/testutils/QueueMapTest.h @@ -0,0 +1,31 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_ +#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_ + +#include +#include +#include "../../../../../implementations/caching/cache/QueueMap.h" +#include "MinimalKeyType.h" +#include "MinimalValueType.h" +#include + +// This class is a parent class for tests on QueueMap. +// It offers functions to work with a QueueMap test object which is built using types having only the minimal type requirements. +// Furthermore, the class checks that there are no memory leaks left after destructing the QueueMap (by counting leftover instances of Keys/Values). +class QueueMapTest: public ::testing::Test { +public: + QueueMapTest(); + ~QueueMapTest(); + + void push(int key, int value); + boost::optional pop(); + boost::optional pop(int key); + boost::optional peek(); + int size(); + +private: + cpputils::unique_ref> _map; +}; + + +#endif diff --git a/test/blockstore/implementations/compressing/CompressingBlockStoreTest.cpp b/test/blockstore/implementations/compressing/CompressingBlockStoreTest.cpp new file mode 100644 index 00000000..99a668e1 --- /dev/null +++ b/test/blockstore/implementations/compressing/CompressingBlockStoreTest.cpp @@ -0,0 +1,28 @@ +#include "../../../implementations/compressing/CompressingBlockStore.h" +#include "../../../implementations/compressing/compressors/Gzip.h" +#include "../../../implementations/compressing/compressors/RunLengthEncoding.h" +#include "../../../implementations/testfake/FakeBlockStore.h" +#include "../../testutils/BlockStoreTest.h" +#include "google/gtest/gtest.h" + +using ::testing::Test; + +using blockstore::BlockStore; +using blockstore::compressing::CompressingBlockStore; +using blockstore::compressing::Gzip; +using blockstore::compressing::RunLengthEncoding; +using blockstore::testfake::FakeBlockStore; + +using cpputils::make_unique_ref; +using cpputils::unique_ref; + +template +class CompressingBlockStoreTestFixture: public BlockStoreTestFixture { +public: + unique_ref createBlockStore() override { + return make_unique_ref>(make_unique_ref()); + } +}; + +INSTANTIATE_TYPED_TEST_CASE_P(Compressing_Gzip, BlockStoreTest, CompressingBlockStoreTestFixture); +INSTANTIATE_TYPED_TEST_CASE_P(Compressing_RunLengthEncoding, BlockStoreTest, CompressingBlockStoreTestFixture); diff --git a/test/blockstore/implementations/compressing/compressors/testutils/CompressorTest.cpp b/test/blockstore/implementations/compressing/compressors/testutils/CompressorTest.cpp new file mode 100644 index 00000000..41fc4f26 --- /dev/null +++ b/test/blockstore/implementations/compressing/compressors/testutils/CompressorTest.cpp @@ -0,0 +1,92 @@ +#include +#include "../../../../../implementations/compressing/compressors/Gzip.h" +#include "../../../../../implementations/compressing/compressors/RunLengthEncoding.h" +#include + +using namespace blockstore::compressing; +using cpputils::Data; +using cpputils::DataFixture; + +template +class CompressorTest: public ::testing::Test { +public: + void EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(const Data &data) { + Data compressed = Compressor::Compress(data); + Data decompressed = Compressor::Decompress(compressed.data(), compressed.size()); + EXPECT_EQ(data, decompressed); + } +}; + +TYPED_TEST_CASE_P(CompressorTest); + +TYPED_TEST_P(CompressorTest, Empty) { + Data empty(0); + this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(empty); +} + +TYPED_TEST_P(CompressorTest, ArbitraryData) { + Data fixture = DataFixture::generate(10240); + this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(fixture); +} + +TYPED_TEST_P(CompressorTest, Zeroes) { + Data zeroes(10240); + zeroes.FillWithZeroes(); + this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(zeroes); +} + +TYPED_TEST_P(CompressorTest, Runs) { + Data data(4096); + std::memset(data.dataOffset(0), 0xF2, 1024); + std::memset(data.dataOffset(1024), 0x00, 1024); + std::memset(data.dataOffset(2048), 0x01, 2048); + this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(data); +} + +TYPED_TEST_P(CompressorTest, RunsAndArbitrary) { + Data data(4096); + std::memset(data.dataOffset(0), 0xF2, 1024); + std::memcpy(data.dataOffset(1024), DataFixture::generate(1024).data(), 1024); + std::memset(data.dataOffset(2048), 0x01, 2048); + this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(data); +} + +TYPED_TEST_P(CompressorTest, LargeData) { + // this is larger than what fits into 16bit (16bit are for example used as run length indicator in RunLengthEncoding) + Data fixture = DataFixture::generate(200000); + this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(fixture); +} + +TYPED_TEST_P(CompressorTest, LargeRuns) { + // each run is larger than what fits into 16bit (16bit are for example used as run length indicator in RunLengthEncoding) + constexpr size_t RUN_SIZE = 200000; + Data data(3*RUN_SIZE); + std::memset(data.dataOffset(0), 0xF2, RUN_SIZE); + std::memset(data.dataOffset(RUN_SIZE), 0x00, RUN_SIZE); + std::memset(data.dataOffset(2*RUN_SIZE), 0x01, RUN_SIZE); + this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(data); +} + +TYPED_TEST_P(CompressorTest, LargeRunsAndArbitrary) { + // each run is larger than what fits into 16bit (16bit are for example used as run length indicator in RunLengthEncoding) + constexpr size_t RUN_SIZE = 200000; + Data data(3*RUN_SIZE); + std::memset(data.dataOffset(0), 0xF2, RUN_SIZE); + std::memcpy(data.dataOffset(RUN_SIZE), DataFixture::generate(RUN_SIZE).data(), RUN_SIZE); + std::memset(data.dataOffset(2*RUN_SIZE), 0x01, RUN_SIZE); + this->EXPECT_COMPRESS_AND_DECOMPRESS_IS_IDENTITY(data); +} + +REGISTER_TYPED_TEST_CASE_P(CompressorTest, + Empty, + ArbitraryData, + Zeroes, + Runs, + RunsAndArbitrary, + LargeData, + LargeRuns, + LargeRunsAndArbitrary +); + +INSTANTIATE_TYPED_TEST_CASE_P(Gzip, CompressorTest, Gzip); +INSTANTIATE_TYPED_TEST_CASE_P(RunLengthEncoding, CompressorTest, RunLengthEncoding); diff --git a/test/blockstore/implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp b/test/blockstore/implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp new file mode 100644 index 00000000..9d3798d9 --- /dev/null +++ b/test/blockstore/implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp @@ -0,0 +1,39 @@ +#include +#include +#include "../../../implementations/encrypted/EncryptedBlockStore.h" +#include "../../../implementations/testfake/FakeBlockStore.h" +#include "../../testutils/BlockStoreTest.h" +#include +#include "google/gtest/gtest.h" + +using ::testing::Test; + +using blockstore::BlockStore; +using blockstore::encrypted::EncryptedBlockStore; +using blockstore::testfake::FakeBlockStore; +using cpputils::AES256_GCM; +using cpputils::AES256_CFB; +using cpputils::FakeAuthenticatedCipher; + +using cpputils::Data; +using cpputils::DataFixture; +using cpputils::make_unique_ref; +using cpputils::unique_ref; + +template +class EncryptedBlockStoreTestFixture: public BlockStoreTestFixture { +public: + unique_ref createBlockStore() override { + return make_unique_ref>(make_unique_ref(), createKeyFixture()); + } + +private: + static typename Cipher::EncryptionKey createKeyFixture(int seed = 0) { + Data data = DataFixture::generate(Cipher::EncryptionKey::BINARY_LENGTH, seed); + return Cipher::EncryptionKey::FromBinary(data.data()); + } +}; + +INSTANTIATE_TYPED_TEST_CASE_P(Encrypted_FakeCipher, BlockStoreTest, EncryptedBlockStoreTestFixture); +INSTANTIATE_TYPED_TEST_CASE_P(Encrypted_AES256_GCM, BlockStoreTest, EncryptedBlockStoreTestFixture); +INSTANTIATE_TYPED_TEST_CASE_P(Encrypted_AES256_CFB, BlockStoreTest, EncryptedBlockStoreTestFixture); diff --git a/test/blockstore/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp b/test/blockstore/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp new file mode 100644 index 00000000..e8ba7372 --- /dev/null +++ b/test/blockstore/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp @@ -0,0 +1,114 @@ +#include +#include +#include "../../../implementations/encrypted/EncryptedBlockStore.h" +#include "../../../implementations/testfake/FakeBlockStore.h" +#include "../../../utils/BlockStoreUtils.h" +#include + +using ::testing::Test; + +using cpputils::DataFixture; +using cpputils::Data; +using cpputils::unique_ref; +using cpputils::make_unique_ref; +using cpputils::FakeAuthenticatedCipher; + +using blockstore::testfake::FakeBlockStore; + +using namespace blockstore::encrypted; + +class EncryptedBlockStoreTest: public Test { +public: + static constexpr unsigned int BLOCKSIZE = 1024; + EncryptedBlockStoreTest(): + baseBlockStore(new FakeBlockStore), + blockStore(make_unique_ref>(std::move(cpputils::nullcheck(std::unique_ptr(baseBlockStore)).value()), FakeAuthenticatedCipher::Key1())), + data(DataFixture::generate(BLOCKSIZE)) { + } + FakeBlockStore *baseBlockStore; + unique_ref> blockStore; + Data data; + + blockstore::Key CreateBlockDirectlyWithFixtureAndReturnKey() { + return blockStore->create(data)->key(); + } + + blockstore::Key CreateBlockWriteFixtureToItAndReturnKey() { + auto block = blockStore->create(Data(data.size())); + block->write(data.data(), 0, data.size()); + return block->key(); + } + + void ModifyBaseBlock(const blockstore::Key &key) { + auto block = baseBlockStore->load(key).value(); + uint8_t middle_byte = ((byte*)block->data())[10]; + uint8_t new_middle_byte = middle_byte + 1; + block->write(&new_middle_byte, 10, 1); + } + + blockstore::Key CopyBaseBlock(const blockstore::Key &key) { + auto source = baseBlockStore->load(key).value(); + return blockstore::utils::copyToNewBlock(baseBlockStore, *source)->key(); + } + +private: + DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStoreTest); +}; + +TEST_F(EncryptedBlockStoreTest, LoadingWithSameKeyWorks_WriteOnCreate) { + auto key = CreateBlockDirectlyWithFixtureAndReturnKey(); + auto loaded = blockStore->load(key); + EXPECT_NE(boost::none, loaded); + EXPECT_EQ(data.size(), (*loaded)->size()); + EXPECT_EQ(0, std::memcmp(data.data(), (*loaded)->data(), data.size())); +} + +TEST_F(EncryptedBlockStoreTest, LoadingWithSameKeyWorks_WriteSeparately) { + auto key = CreateBlockWriteFixtureToItAndReturnKey(); + auto loaded = blockStore->load(key); + EXPECT_NE(boost::none, loaded); + EXPECT_EQ(data.size(), (*loaded)->size()); + EXPECT_EQ(0, std::memcmp(data.data(), (*loaded)->data(), data.size())); +} + +TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentKeyDoesntWork_WriteOnCreate) { + auto key = CreateBlockDirectlyWithFixtureAndReturnKey(); + blockStore->__setKey(FakeAuthenticatedCipher::Key2()); + auto loaded = blockStore->load(key); + EXPECT_EQ(boost::none, loaded); +} + +TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentKeyDoesntWork_WriteSeparately) { + auto key = CreateBlockWriteFixtureToItAndReturnKey(); + blockStore->__setKey(FakeAuthenticatedCipher::Key2()); + auto loaded = blockStore->load(key); + EXPECT_EQ(boost::none, loaded); +} + +TEST_F(EncryptedBlockStoreTest, LoadingModifiedBlockFails_WriteOnCreate) { + auto key = CreateBlockDirectlyWithFixtureAndReturnKey(); + ModifyBaseBlock(key); + auto loaded = blockStore->load(key); + EXPECT_EQ(boost::none, loaded); +} + +TEST_F(EncryptedBlockStoreTest, LoadingModifiedBlockFails_WriteSeparately) { + auto key = CreateBlockWriteFixtureToItAndReturnKey(); + ModifyBaseBlock(key); + auto loaded = blockStore->load(key); + EXPECT_EQ(boost::none, loaded); +} + +TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentBlockIdFails_WriteOnCreate) { + auto key = CreateBlockDirectlyWithFixtureAndReturnKey(); + auto key2 = CopyBaseBlock(key); + auto loaded = blockStore->load(key2); + EXPECT_EQ(boost::none, loaded); +} + +TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentBlockIdFails_WriteSeparately) { + auto key = CreateBlockWriteFixtureToItAndReturnKey(); + auto key2 = CopyBaseBlock(key); + auto loaded = blockStore->load(key2); + EXPECT_EQ(boost::none, loaded); +} diff --git a/test/blockstore/implementations/inmemory/InMemoryBlockStoreTest.cpp b/test/blockstore/implementations/inmemory/InMemoryBlockStoreTest.cpp new file mode 100644 index 00000000..6c4c1fe5 --- /dev/null +++ b/test/blockstore/implementations/inmemory/InMemoryBlockStoreTest.cpp @@ -0,0 +1,31 @@ +#include "../../../implementations/inmemory/InMemoryBlock.h" +#include "../../../implementations/inmemory/InMemoryBlockStore.h" +#include "../../testutils/BlockStoreTest.h" +#include "../../testutils/BlockStoreWithRandomKeysTest.h" +#include "google/gtest/gtest.h" +#include + + +using blockstore::BlockStore; +using blockstore::BlockStoreWithRandomKeys; +using blockstore::inmemory::InMemoryBlockStore; +using cpputils::unique_ref; +using cpputils::make_unique_ref; + +class InMemoryBlockStoreTestFixture: public BlockStoreTestFixture { +public: + unique_ref createBlockStore() override { + return make_unique_ref(); + } +}; + +INSTANTIATE_TYPED_TEST_CASE_P(InMemory, BlockStoreTest, InMemoryBlockStoreTestFixture); + +class InMemoryBlockStoreWithRandomKeysTestFixture: public BlockStoreWithRandomKeysTestFixture { +public: + unique_ref createBlockStore() override { + return make_unique_ref(); + } +}; + +INSTANTIATE_TYPED_TEST_CASE_P(InMemory, BlockStoreWithRandomKeysTest, InMemoryBlockStoreWithRandomKeysTestFixture); diff --git a/test/blockstore/implementations/ondisk/OnDiskBlockStoreTest.cpp b/test/blockstore/implementations/ondisk/OnDiskBlockStoreTest.cpp new file mode 100644 index 00000000..fc59b57b --- /dev/null +++ b/test/blockstore/implementations/ondisk/OnDiskBlockStoreTest.cpp @@ -0,0 +1,42 @@ +#include "../../../implementations/ondisk/OnDiskBlock.h" +#include "../../../implementations/ondisk/OnDiskBlockStore.h" +#include "../../testutils/BlockStoreTest.h" +#include "../../testutils/BlockStoreWithRandomKeysTest.h" +#include "google/gtest/gtest.h" + +#include "messmer/cpp-utils/tempfile/TempDir.h" + + +using blockstore::BlockStore; +using blockstore::BlockStoreWithRandomKeys; +using blockstore::ondisk::OnDiskBlockStore; + +using cpputils::TempDir; +using cpputils::unique_ref; +using cpputils::make_unique_ref; + +class OnDiskBlockStoreTestFixture: public BlockStoreTestFixture { +public: + OnDiskBlockStoreTestFixture(): tempdir() {} + + unique_ref createBlockStore() override { + return make_unique_ref(tempdir.path()); + } +private: + TempDir tempdir; +}; + +INSTANTIATE_TYPED_TEST_CASE_P(OnDisk, BlockStoreTest, OnDiskBlockStoreTestFixture); + +class OnDiskBlockStoreWithRandomKeysTestFixture: public BlockStoreWithRandomKeysTestFixture { +public: + OnDiskBlockStoreWithRandomKeysTestFixture(): tempdir() {} + + unique_ref createBlockStore() override { + return make_unique_ref(tempdir.path()); + } +private: + TempDir tempdir; +}; + +INSTANTIATE_TYPED_TEST_CASE_P(OnDisk, BlockStoreWithRandomKeysTest, OnDiskBlockStoreWithRandomKeysTestFixture); diff --git a/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp b/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp new file mode 100644 index 00000000..06b5556a --- /dev/null +++ b/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp @@ -0,0 +1,83 @@ +#include "../../../../implementations/ondisk/OnDiskBlock.h" +#include "google/gtest/gtest.h" + +#include +#include + +using ::testing::Test; +using ::testing::WithParamInterface; +using ::testing::Values; +using cpputils::Data; +using cpputils::TempFile; +using cpputils::TempDir; +using cpputils::unique_ref; + +using namespace blockstore; +using namespace blockstore::ondisk; + +namespace bf = boost::filesystem; + +class OnDiskBlockCreateTest: public Test { +public: + OnDiskBlockCreateTest() + // Don't create the temp file yet (therefore pass false to the TempFile constructor) + : dir(), + key(Key::FromString("1491BB4932A389EE14BC7090AC772972")), + file(dir.path() / key.ToString(), false) { + } + TempDir dir; + Key key; + TempFile file; +}; + +TEST_F(OnDiskBlockCreateTest, CreatingBlockCreatesFile) { + EXPECT_FALSE(bf::exists(file.path())); + + auto block = OnDiskBlock::CreateOnDisk(dir.path(), key, Data(0)); + + EXPECT_TRUE(bf::exists(file.path())); + EXPECT_TRUE(bf::is_regular_file(file.path())); +} + +TEST_F(OnDiskBlockCreateTest, CreatingExistingBlockReturnsNull) { + auto block1 = OnDiskBlock::CreateOnDisk(dir.path(), key, Data(0)); + auto block2 = OnDiskBlock::CreateOnDisk(dir.path(), key, Data(0)); + EXPECT_TRUE((bool)block1); + EXPECT_FALSE((bool)block2); +} + +class OnDiskBlockCreateSizeTest: public OnDiskBlockCreateTest, public WithParamInterface { +public: + unique_ref block; + Data ZEROES; + + OnDiskBlockCreateSizeTest(): + block(OnDiskBlock::CreateOnDisk(dir.path(), key, std::move(Data(GetParam()).FillWithZeroes())).value()), + ZEROES(block->size()) + { + ZEROES.FillWithZeroes(); + } +}; +INSTANTIATE_TEST_CASE_P(OnDiskBlockCreateSizeTest, OnDiskBlockCreateSizeTest, Values(0, 1, 5, 1024, 10*1024*1024)); + +TEST_P(OnDiskBlockCreateSizeTest, OnDiskSizeIsCorrect) { + Data fileContent = Data::LoadFromFile(file.path()).value(); + EXPECT_EQ(GetParam(), fileContent.size()); +} + +TEST_P(OnDiskBlockCreateSizeTest, OnDiskBlockIsZeroedOut) { + Data fileContent = Data::LoadFromFile(file.path()).value(); + EXPECT_EQ(ZEROES, fileContent); +} + +// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface. +// Here, we create it using OnDiskBlock::CreateOnDisk() +TEST_P(OnDiskBlockCreateSizeTest, InMemorySizeIsCorrect) { + EXPECT_EQ(GetParam(), block->size()); +} + +// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface. +// Here, we create it using OnDiskBlock::CreateOnDisk() +TEST_P(OnDiskBlockCreateSizeTest, InMemoryBlockIsZeroedOut) { + EXPECT_EQ(0, std::memcmp(ZEROES.data(), block->data(), block->size())); +} diff --git a/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp b/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp new file mode 100644 index 00000000..619924b2 --- /dev/null +++ b/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp @@ -0,0 +1,115 @@ +#include "../../../../implementations/ondisk/OnDiskBlock.h" +#include +#include "google/gtest/gtest.h" + +#include +#include + +using ::testing::Test; +using ::testing::WithParamInterface; +using ::testing::Values; + +using cpputils::Data; +using cpputils::DataFixture; +using cpputils::TempFile; +using cpputils::TempDir; +using cpputils::unique_ref; + +using namespace blockstore; +using namespace blockstore::ondisk; + +namespace bf = boost::filesystem; + +class OnDiskBlockFlushTest: public Test, public WithParamInterface { +public: + OnDiskBlockFlushTest() + // Don't create the temp file yet (therefore pass false to the TempFile constructor) + : dir(), + key(Key::FromString("1491BB4932A389EE14BC7090AC772972")), + file(dir.path() / key.ToString(), false), + randomData(DataFixture::generate(GetParam())) { + } + TempDir dir; + Key key; + TempFile file; + + Data randomData; + + unique_ref CreateBlockAndLoadItFromDisk() { + { + OnDiskBlock::CreateOnDisk(dir.path(), key, randomData.copy()).value(); + } + return OnDiskBlock::LoadFromDisk(dir.path(), key).value(); + } + + unique_ref CreateBlock() { + return OnDiskBlock::CreateOnDisk(dir.path(), key, randomData.copy()).value(); + } + + void WriteDataToBlock(const unique_ref &block) { + block->write(randomData.data(), 0, randomData.size()); + } + + void EXPECT_BLOCK_DATA_CORRECT(const unique_ref &block) { + EXPECT_EQ(randomData.size(), block->size()); + EXPECT_EQ(0, std::memcmp(randomData.data(), block->data(), randomData.size())); + } + + void EXPECT_STORED_FILE_DATA_CORRECT() { + Data actual = Data::LoadFromFile(file.path()).value(); + EXPECT_EQ(randomData, actual); + } +}; +INSTANTIATE_TEST_CASE_P(OnDiskBlockFlushTest, OnDiskBlockFlushTest, Values((size_t)0, (size_t)1, (size_t)1024, (size_t)4096, (size_t)10*1024*1024)); + +// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface. +// Here, we create it using OnDiskBlock::CreateOnDisk() +TEST_P(OnDiskBlockFlushTest, AfterCreate_FlushingDoesntChangeBlock) { + auto block = CreateBlock(); + WriteDataToBlock(block); + + EXPECT_BLOCK_DATA_CORRECT(block); +} + +// This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface. +// Here, we create it using OnDiskBlock::CreateOnDisk() / OnDiskBlock::LoadFromDisk() +TEST_P(OnDiskBlockFlushTest, AfterLoad_FlushingDoesntChangeBlock) { + auto block = CreateBlockAndLoadItFromDisk(); + WriteDataToBlock(block); + + EXPECT_BLOCK_DATA_CORRECT(block); +} + +TEST_P(OnDiskBlockFlushTest, AfterCreate_FlushingWritesCorrectData) { + auto block = CreateBlock(); + WriteDataToBlock(block); + + EXPECT_STORED_FILE_DATA_CORRECT(); +} + +TEST_P(OnDiskBlockFlushTest, AfterLoad_FlushingWritesCorrectData) { + auto block = CreateBlockAndLoadItFromDisk(); + WriteDataToBlock(block); + + EXPECT_STORED_FILE_DATA_CORRECT(); +} + +// This test is also tested by OnDiskBlockStoreTest, but there it can only checks block content by loading it again. +// Here, we check the content on disk. +TEST_P(OnDiskBlockFlushTest, AfterCreate_FlushesWhenDestructed) { + { + auto block = CreateBlock(); + WriteDataToBlock(block); + } + EXPECT_STORED_FILE_DATA_CORRECT(); +} + +// This test is also tested by OnDiskBlockStoreTest, but there it can only checks block content by loading it again. +// Here, we check the content on disk. +TEST_P(OnDiskBlockFlushTest, AfterLoad_FlushesWhenDestructed) { + { + auto block = CreateBlockAndLoadItFromDisk(); + WriteDataToBlock(block); + } + EXPECT_STORED_FILE_DATA_CORRECT(); +} diff --git a/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp b/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp new file mode 100644 index 00000000..1dc00625 --- /dev/null +++ b/test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp @@ -0,0 +1,80 @@ +#include "../../../../implementations/ondisk/OnDiskBlock.h" +#include +#include "../../../../utils/FileDoesntExistException.h" +#include "google/gtest/gtest.h" + +#include +#include +#include +#include +#include + +using ::testing::Test; +using ::testing::WithParamInterface; +using ::testing::Values; + +using std::ofstream; +using std::ios; +using cpputils::Data; +using cpputils::DataFixture; +using cpputils::TempFile; +using cpputils::TempDir; +using cpputils::unique_ref; + +using namespace blockstore; +using namespace blockstore::ondisk; + +namespace bf = boost::filesystem; + +class OnDiskBlockLoadTest: public Test, public WithParamInterface { +public: + OnDiskBlockLoadTest(): + dir(), + key(Key::FromString("1491BB4932A389EE14BC7090AC772972")), + file(dir.path() / key.ToString()) { + } + TempDir dir; + Key key; + TempFile file; + + void SetFileSize(size_t size) { + Data data(size); + data.StoreToFile(file.path()); + } + + void StoreData(const Data &data) { + data.StoreToFile(file.path()); + } + + unique_ref LoadBlock() { + return OnDiskBlock::LoadFromDisk(dir.path(), key).value(); + } + + void EXPECT_BLOCK_DATA_EQ(const Data &expected, const OnDiskBlock &actual) { + EXPECT_EQ(expected.size(), actual.size()); + EXPECT_EQ(0, std::memcmp(expected.data(), actual.data(), expected.size())); + } +}; +INSTANTIATE_TEST_CASE_P(OnDiskBlockLoadTest, OnDiskBlockLoadTest, Values(0, 1, 5, 1024, 10*1024*1024)); + +TEST_P(OnDiskBlockLoadTest, FileSizeIsCorrect) { + SetFileSize(GetParam()); + + auto block = LoadBlock(); + + EXPECT_EQ(GetParam(), block->size()); +} + +TEST_P(OnDiskBlockLoadTest, LoadedDataIsCorrect) { + Data randomData = DataFixture::generate(GetParam()); + StoreData(randomData); + + auto block = LoadBlock(); + + EXPECT_BLOCK_DATA_EQ(randomData, *block); +} + +TEST_F(OnDiskBlockLoadTest, LoadNotExistingBlock) { + Key key2 = Key::FromString("272EE5517627CFA147A971A8E6E747E0"); + EXPECT_EQ(boost::none, OnDiskBlock::LoadFromDisk(dir.path(), key2)); +} diff --git a/test/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreTest.cpp b/test/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreTest.cpp new file mode 100644 index 00000000..c3647a1e --- /dev/null +++ b/test/blockstore/implementations/parallelaccess/ParallelAccessBlockStoreTest.cpp @@ -0,0 +1,23 @@ +#include "../../../implementations/parallelaccess/ParallelAccessBlockStore.h" +#include "../../../implementations/testfake/FakeBlockStore.h" +#include "../../testutils/BlockStoreTest.h" +#include "google/gtest/gtest.h" + + +using blockstore::BlockStore; +using blockstore::parallelaccess::ParallelAccessBlockStore; +using blockstore::testfake::FakeBlockStore; + +using cpputils::make_unique_ref; +using cpputils::unique_ref; + +class ParallelAccessBlockStoreTestFixture: public BlockStoreTestFixture { +public: + unique_ref createBlockStore() override { + return make_unique_ref(make_unique_ref()); + } +}; + +INSTANTIATE_TYPED_TEST_CASE_P(ParallelAccess, BlockStoreTest, ParallelAccessBlockStoreTestFixture); + +//TODO Add specific tests ensuring that loading the same block twice doesn't load it twice from the underlying blockstore diff --git a/test/blockstore/implementations/testfake/TestFakeBlockStoreTest.cpp b/test/blockstore/implementations/testfake/TestFakeBlockStoreTest.cpp new file mode 100644 index 00000000..55e04577 --- /dev/null +++ b/test/blockstore/implementations/testfake/TestFakeBlockStoreTest.cpp @@ -0,0 +1,30 @@ +#include "../../../implementations/testfake/FakeBlock.h" +#include "../../../implementations/testfake/FakeBlockStore.h" +#include "../../testutils/BlockStoreTest.h" +#include "../../testutils/BlockStoreWithRandomKeysTest.h" +#include "google/gtest/gtest.h" + + +using blockstore::BlockStore; +using blockstore::BlockStoreWithRandomKeys; +using blockstore::testfake::FakeBlockStore; +using cpputils::unique_ref; +using cpputils::make_unique_ref; + +class FakeBlockStoreTestFixture: public BlockStoreTestFixture { +public: + unique_ref createBlockStore() override { + return make_unique_ref(); + } +}; + +INSTANTIATE_TYPED_TEST_CASE_P(TestFake, BlockStoreTest, FakeBlockStoreTestFixture); + +class FakeBlockStoreWithRandomKeysTestFixture: public BlockStoreWithRandomKeysTestFixture { +public: + unique_ref createBlockStore() override { + return make_unique_ref(); + } +}; + +INSTANTIATE_TYPED_TEST_CASE_P(TestFake, BlockStoreWithRandomKeysTest, FakeBlockStoreWithRandomKeysTestFixture); diff --git a/test/blockstore/interface/BlockStoreTest.cpp b/test/blockstore/interface/BlockStoreTest.cpp new file mode 100644 index 00000000..c452c306 --- /dev/null +++ b/test/blockstore/interface/BlockStoreTest.cpp @@ -0,0 +1,4 @@ +/* + * Tests that the header can be included without needing additional header includes as dependencies. + */ +#include "../../interface/BlockStore.h" diff --git a/test/blockstore/interface/BlockTest.cpp b/test/blockstore/interface/BlockTest.cpp new file mode 100644 index 00000000..a2d0f322 --- /dev/null +++ b/test/blockstore/interface/BlockTest.cpp @@ -0,0 +1,4 @@ +/* + * Tests that the header can be included without needing additional header includes as dependencies. + */ +#include "../../interface/Block.h" diff --git a/test/blockstore/interface/helpers/BlockStoreWithRandomKeysTest.cpp b/test/blockstore/interface/helpers/BlockStoreWithRandomKeysTest.cpp new file mode 100644 index 00000000..fd9fe959 --- /dev/null +++ b/test/blockstore/interface/helpers/BlockStoreWithRandomKeysTest.cpp @@ -0,0 +1,143 @@ +#include "../../../interface/helpers/BlockStoreWithRandomKeys.h" +#include "google/gtest/gtest.h" +#include "google/gmock/gmock.h" +#include + +using ::testing::Test; +using ::testing::_; +using ::testing::Return; +using ::testing::Invoke; +using ::testing::Eq; +using ::testing::ByRef; + +using std::string; +using cpputils::Data; +using cpputils::DataFixture; +using cpputils::unique_ref; +using boost::optional; + +using namespace blockstore; + +class BlockStoreWithRandomKeysMock: public BlockStoreWithRandomKeys { +public: + optional> tryCreate(const Key &key, Data data) { + return cpputils::nullcheck(std::unique_ptr(do_create(key, data))); + } + MOCK_METHOD2(do_create, Block*(const Key &, const Data &data)); + optional> load(const Key &key) { + return cpputils::nullcheck(std::unique_ptr(do_load(key))); + } + MOCK_METHOD1(do_load, Block*(const Key &)); + void remove(unique_ref block) {UNUSED(block);} + MOCK_CONST_METHOD0(numBlocks, uint64_t()); +}; + +class BlockMock: public Block { +public: + BlockMock(): Block(cpputils::Random::PseudoRandom().getFixedSize()) {} + MOCK_CONST_METHOD0(data, const void*()); + MOCK_METHOD3(write, void(const void*, uint64_t, uint64_t)); + MOCK_METHOD0(flush, void()); + MOCK_CONST_METHOD0(size, size_t()); + MOCK_METHOD1(resize, void(size_t)); + MOCK_CONST_METHOD0(key, const Key&()); +}; + +class BlockStoreWithRandomKeysTest: public Test { +public: + BlockStoreWithRandomKeysTest() :blockStoreMock(), blockStore(blockStoreMock), + key(Key::FromString("1491BB4932A389EE14BC7090AC772972")) {} + + BlockStoreWithRandomKeysMock blockStoreMock; + BlockStore &blockStore; + const blockstore::Key key; + + Data createDataWithSize(size_t size) { + Data fixture(DataFixture::generate(size)); + Data data(size); + std::memcpy(data.data(), fixture.data(), size); + return data; + } +}; + +TEST_F(BlockStoreWithRandomKeysTest, DataIsPassedThrough0) { + Data data = createDataWithSize(0); + EXPECT_CALL(blockStoreMock, do_create(_, Eq(ByRef(data)))).WillOnce(Return(new BlockMock)); + blockStore.create(data); +} + +TEST_F(BlockStoreWithRandomKeysTest, DataIsPassedThrough1) { + Data data = createDataWithSize(1); + EXPECT_CALL(blockStoreMock, do_create(_, Eq(ByRef(data)))).WillOnce(Return(new BlockMock)); + blockStore.create(data); +} + +TEST_F(BlockStoreWithRandomKeysTest, DataIsPassedThrough1024) { + Data data = createDataWithSize(1024); + EXPECT_CALL(blockStoreMock, do_create(_, Eq(ByRef(data)))).WillOnce(Return(new BlockMock)); + blockStore.create(data); +} + +TEST_F(BlockStoreWithRandomKeysTest, KeyHasCorrectSize) { + EXPECT_CALL(blockStoreMock, do_create(_, _)).WillOnce(Invoke([](const Key &key, const Data &) { + EXPECT_EQ(Key::STRING_LENGTH, key.ToString().size()); + return new BlockMock; + })); + + blockStore.create(createDataWithSize(1024)); +} + +TEST_F(BlockStoreWithRandomKeysTest, TwoBlocksGetDifferentKeys) { + Key first_key = key; + EXPECT_CALL(blockStoreMock, do_create(_, _)) + .WillOnce(Invoke([&first_key](const Key &key, const Data &) { + first_key = key; + return new BlockMock; + })) + .WillOnce(Invoke([&first_key](const Key &key, const Data &) { + EXPECT_NE(first_key, key); + return new BlockMock; + })); + + Data data = createDataWithSize(1024); + blockStore.create(data); + blockStore.create(data); +} + +TEST_F(BlockStoreWithRandomKeysTest, WillTryADifferentKeyIfKeyAlreadyExists) { + Key first_key = key; + Data data = createDataWithSize(1024); + EXPECT_CALL(blockStoreMock, do_create(_, Eq(ByRef(data)))) + .WillOnce(Invoke([&first_key](const Key &key, const Data &) { + first_key = key; + return nullptr; + })) + //TODO Check that this test case fails when the second do_create call gets different data + .WillOnce(Invoke([&first_key](const Key &key, const Data &) { + EXPECT_NE(first_key, key); + return new BlockMock; + })); + + blockStore.create(data); +} + +TEST_F(BlockStoreWithRandomKeysTest, WillTryADifferentKeyIfKeyAlreadyExistsTwoTimes) { + Key first_key = key; + Data data = createDataWithSize(1024); + EXPECT_CALL(blockStoreMock, do_create(_, Eq(ByRef(data)))) + .WillOnce(Invoke([&first_key](const Key &key, const Data &) { + first_key = key; + return nullptr; + })) + //TODO Check that this test case fails when the second/third do_create calls get different data + .WillOnce(Invoke([&first_key](const Key &key, const Data &) { + first_key = key; + return nullptr; + })) + .WillOnce(Invoke([&first_key](const Key &key, const Data &) { + EXPECT_NE(first_key, key); + return new BlockMock; + })); + + blockStore.create(data); +} diff --git a/test/blockstore/testutils/BlockStoreTest.h b/test/blockstore/testutils/BlockStoreTest.h new file mode 100644 index 00000000..91980125 --- /dev/null +++ b/test/blockstore/testutils/BlockStoreTest.h @@ -0,0 +1,148 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_TESTUTILS_BLOCKSTORETEST_H_ +#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_TESTUTILS_BLOCKSTORETEST_H_ + +#include "google/gtest/gtest.h" + +#include "../../interface/BlockStore.h" + +class BlockStoreTestFixture { +public: + virtual ~BlockStoreTestFixture() {} + virtual cpputils::unique_ref createBlockStore() = 0; +}; + +template +class BlockStoreTest: public ::testing::Test { +public: + BlockStoreTest() :fixture() {} + + BOOST_STATIC_ASSERT_MSG( + (std::is_base_of::value), + "Given test fixture for instantiating the (type parameterized) BlockStoreTest must inherit from BlockStoreTestFixture" + ); + + ConcreteBlockStoreTestFixture fixture; +}; + +TYPED_TEST_CASE_P(BlockStoreTest); + +TYPED_TEST_P(BlockStoreTest, TwoCreatedBlocksHaveDifferentKeys) { + auto blockStore = this->fixture.createBlockStore(); + auto block1 = blockStore->create(cpputils::Data(1024)); + auto block2 = blockStore->create(cpputils::Data(1024)); + EXPECT_NE(block1->key(), block2->key()); +} + +TYPED_TEST_P(BlockStoreTest, BlockIsNotLoadableAfterDeleting) { + auto blockStore = this->fixture.createBlockStore(); + auto blockkey = blockStore->create(cpputils::Data(1024))->key(); + auto block = blockStore->load(blockkey); + EXPECT_NE(boost::none, block); + blockStore->remove(std::move(*block)); + EXPECT_EQ(boost::none, blockStore->load(blockkey)); +} + +TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectOnEmptyBlockstore) { + auto blockStore = this->fixture.createBlockStore(); + EXPECT_EQ(0, blockStore->numBlocks()); +} + +TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingOneBlock) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->create(cpputils::Data(1)); + EXPECT_EQ(1, blockStore->numBlocks()); +} + +TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingOneBlock_AfterClosingBlock) { + auto blockStore = this->fixture.createBlockStore(); + blockStore->create(cpputils::Data(1)); + EXPECT_EQ(1, blockStore->numBlocks()); +} + +TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterRemovingTheLastBlock) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->create(cpputils::Data(1)); + blockStore->remove(std::move(block)); + EXPECT_EQ(0, blockStore->numBlocks()); +} + +TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks) { + auto blockStore = this->fixture.createBlockStore(); + auto block1 = blockStore->create(cpputils::Data(1)); + auto block2 = blockStore->create(cpputils::Data(0)); + EXPECT_EQ(2, blockStore->numBlocks()); +} + +TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingFirstBlock) { + auto blockStore = this->fixture.createBlockStore(); + blockStore->create(cpputils::Data(1)); + auto block2 = blockStore->create(cpputils::Data(0)); + EXPECT_EQ(2, blockStore->numBlocks()); +} + +TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingSecondBlock) { + auto blockStore = this->fixture.createBlockStore(); + auto block1 = blockStore->create(cpputils::Data(1)); + blockStore->create(cpputils::Data(0)); + EXPECT_EQ(2, blockStore->numBlocks()); +} + +TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingBothBlocks) { + auto blockStore = this->fixture.createBlockStore(); + blockStore->create(cpputils::Data(1)); + blockStore->create(cpputils::Data(0)); + EXPECT_EQ(2, blockStore->numBlocks()); +} + +TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterRemovingABlock) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->create(cpputils::Data(1)); + blockStore->create(cpputils::Data(1)); + blockStore->remove(std::move(block)); + EXPECT_EQ(1, blockStore->numBlocks()); +} + +TYPED_TEST_P(BlockStoreTest, CanRemoveModifiedBlock) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->create(cpputils::Data(5)); + block->write("data", 0, 4); + blockStore->remove(std::move(block)); + EXPECT_EQ(0, blockStore->numBlocks()); +} + +#include "BlockStoreTest_Size.h" +#include "BlockStoreTest_Data.h" + + +REGISTER_TYPED_TEST_CASE_P(BlockStoreTest, + CreatedBlockHasCorrectSize, + LoadingUnchangedBlockHasCorrectSize, + CreatedBlockData, + LoadingUnchangedBlockData, + LoadedBlockIsCorrect, +// LoadedBlockIsCorrectWhenLoadedDirectlyAfterFlushing, + AfterCreate_FlushingDoesntChangeBlock, + AfterLoad_FlushingDoesntChangeBlock, + AfterCreate_FlushesWhenDestructed, + AfterLoad_FlushesWhenDestructed, + LoadNonExistingBlock, + TwoCreatedBlocksHaveDifferentKeys, + BlockIsNotLoadableAfterDeleting, + NumBlocksIsCorrectOnEmptyBlockstore, + NumBlocksIsCorrectAfterAddingOneBlock, + NumBlocksIsCorrectAfterAddingOneBlock_AfterClosingBlock, + NumBlocksIsCorrectAfterRemovingTheLastBlock, + NumBlocksIsCorrectAfterAddingTwoBlocks, + NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingFirstBlock, + NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingSecondBlock, + NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingBothBlocks, + NumBlocksIsCorrectAfterRemovingABlock, + WriteAndReadImmediately, + WriteAndReadAfterLoading, + OverwriteAndRead, + CanRemoveModifiedBlock +); + + +#endif diff --git a/test/blockstore/testutils/BlockStoreTest_Data.h b/test/blockstore/testutils/BlockStoreTest_Data.h new file mode 100644 index 00000000..be0e4f1f --- /dev/null +++ b/test/blockstore/testutils/BlockStoreTest_Data.h @@ -0,0 +1,107 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTORETEST_DATA_H_ +#define MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTORETEST_DATA_H_ + +// This file is meant to be included by BlockStoreTest.h only + +struct DataRange { + size_t blocksize; + off_t offset; + size_t count; +}; + +class BlockStoreDataParametrizedTest { +public: + BlockStoreDataParametrizedTest(cpputils::unique_ref blockStore_, const DataRange &testData_) + : blockStore(std::move(blockStore_)), + testData(testData_), + foregroundData(cpputils::DataFixture::generate(testData.count, 0)), + backgroundData(cpputils::DataFixture::generate(testData.blocksize, 1)) { + } + + void TestWriteAndReadImmediately() { + auto block = blockStore->create(cpputils::Data(testData.blocksize).FillWithZeroes()); + block->write(foregroundData.data(), testData.offset, testData.count); + + EXPECT_DATA_READS_AS(foregroundData, *block, testData.offset, testData.count); + EXPECT_DATA_IS_ZEROES_OUTSIDE_OF(*block, testData.offset, testData.count); + } + + void TestWriteAndReadAfterLoading() { + blockstore::Key key = CreateBlockWriteToItAndReturnKey(foregroundData); + + auto loaded_block = blockStore->load(key).value(); + EXPECT_DATA_READS_AS(foregroundData, *loaded_block, testData.offset, testData.count); + EXPECT_DATA_IS_ZEROES_OUTSIDE_OF(*loaded_block, testData.offset, testData.count); + } + + void TestOverwriteAndRead() { + auto block = blockStore->create(cpputils::Data(testData.blocksize)); + block->write(backgroundData.data(), 0, testData.blocksize); + block->write(foregroundData.data(), testData.offset, testData.count); + EXPECT_DATA_READS_AS(foregroundData, *block, testData.offset, testData.count); + EXPECT_DATA_READS_AS_OUTSIDE_OF(backgroundData, *block, testData.offset, testData.count); + } + +private: + cpputils::unique_ref blockStore; + DataRange testData; + cpputils::Data foregroundData; + cpputils::Data backgroundData; + + blockstore::Key CreateBlockWriteToItAndReturnKey(const cpputils::Data &to_write) { + auto newblock = blockStore->create(cpputils::Data(testData.blocksize).FillWithZeroes()); + + newblock->write(to_write.data(), testData.offset, testData.count); + return newblock->key(); + } + + void EXPECT_DATA_READS_AS(const cpputils::Data &expected, const blockstore::Block &block, off_t offset, size_t count) { + cpputils::Data read(count); + std::memcpy(read.data(), (uint8_t*)block.data() + offset, count); + EXPECT_EQ(expected, read); + } + + void EXPECT_DATA_READS_AS_OUTSIDE_OF(const cpputils::Data &expected, const blockstore::Block &block, off_t start, size_t count) { + cpputils::Data begin(start); + cpputils::Data end(testData.blocksize - count - start); + + std::memcpy(begin.data(), expected.data(), start); + std::memcpy(end.data(), (uint8_t*)expected.data()+start+count, end.size()); + + EXPECT_DATA_READS_AS(begin, block, 0, start); + EXPECT_DATA_READS_AS(end, block, start + count, end.size()); + } + + void EXPECT_DATA_IS_ZEROES_OUTSIDE_OF(const blockstore::Block &block, off_t start, size_t count) { + cpputils::Data ZEROES(testData.blocksize); + ZEROES.FillWithZeroes(); + EXPECT_DATA_READS_AS_OUTSIDE_OF(ZEROES, block, start, count); + } +}; + +inline std::vector DATA_RANGES() { + return { + DataRange{1024, 0, 1024}, // full size leaf, access beginning to end + DataRange{1024, 100, 1024 - 200}, // full size leaf, access middle to middle + DataRange{1024, 0, 1024 - 100}, // full size leaf, access beginning to middle + DataRange{1024, 100, 1024 - 100}, // full size leaf, access middle to end + DataRange{1024 - 100, 0, 1024 - 100}, // non-full size leaf, access beginning to end + DataRange{1024 - 100, 100, 1024 - 300}, // non-full size leaf, access middle to middle + DataRange{1024 - 100, 0, 1024 - 200}, // non-full size leaf, access beginning to middle + DataRange{1024 - 100, 100, 1024 - 200} // non-full size leaf, access middle to end + }; +}; +#define TYPED_TEST_P_FOR_ALL_DATA_RANGES(TestName) \ + TYPED_TEST_P(BlockStoreTest, TestName) { \ + for (auto dataRange: DATA_RANGES()) { \ + BlockStoreDataParametrizedTest(this->fixture.createBlockStore(), dataRange) \ + .Test##TestName(); \ + } \ + } + +TYPED_TEST_P_FOR_ALL_DATA_RANGES(WriteAndReadImmediately); +TYPED_TEST_P_FOR_ALL_DATA_RANGES(WriteAndReadAfterLoading); +TYPED_TEST_P_FOR_ALL_DATA_RANGES(OverwriteAndRead); + +#endif diff --git a/test/blockstore/testutils/BlockStoreTest_Size.h b/test/blockstore/testutils/BlockStoreTest_Size.h new file mode 100644 index 00000000..d233d7d9 --- /dev/null +++ b/test/blockstore/testutils/BlockStoreTest_Size.h @@ -0,0 +1,164 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTORETEST_SIZE_H_ +#define MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTORETEST_SIZE_H_ + +// This file is meant to be included by BlockStoreTest.h only + +#include +#include + +class BlockStoreSizeParameterizedTest { +public: + BlockStoreSizeParameterizedTest(cpputils::unique_ref blockStore_, size_t size_): blockStore(std::move(blockStore_)), size(size_) {} + + void TestCreatedBlockHasCorrectSize() { + auto block = CreateBlock(); + EXPECT_EQ(size, block->size()); + } + + void TestLoadingUnchangedBlockHasCorrectSize() { + blockstore::Key key = CreateBlock()->key(); + auto loaded_block = blockStore->load(key).value(); + EXPECT_EQ(size, loaded_block->size()); + } + + void TestCreatedBlockData() { + cpputils::Data data = cpputils::DataFixture::generate(size); + auto block = blockStore->create(data); + EXPECT_EQ(0, std::memcmp(data.data(), block->data(), size)); + } + + void TestLoadingUnchangedBlockData() { + cpputils::Data data = cpputils::DataFixture::generate(size); + blockstore::Key key = blockStore->create(data)->key(); + auto loaded_block = blockStore->load(key).value(); + EXPECT_EQ(0, std::memcmp(data.data(), loaded_block->data(), size)); + } + + void TestLoadedBlockIsCorrect() { + cpputils::Data randomData = cpputils::DataFixture::generate(size); + auto loaded_block = StoreDataToBlockAndLoadIt(randomData); + EXPECT_EQ(size, loaded_block->size()); + EXPECT_EQ(0, std::memcmp(randomData.data(), loaded_block->data(), size)); + } + + void TestLoadedBlockIsCorrectWhenLoadedDirectlyAfterFlushing() { + cpputils::Data randomData = cpputils::DataFixture::generate(size); + auto loaded_block = StoreDataToBlockAndLoadItDirectlyAfterFlushing(randomData); + EXPECT_EQ(size, loaded_block->size()); + EXPECT_EQ(0, std::memcmp(randomData.data(), loaded_block->data(), size)); + } + + void TestAfterCreate_FlushingDoesntChangeBlock() { + cpputils::Data randomData = cpputils::DataFixture::generate(size); + auto block = CreateBlock(); + WriteDataToBlock(block.get(), randomData); + block->flush(); + + EXPECT_BLOCK_DATA_CORRECT(*block, randomData); + } + + void TestAfterLoad_FlushingDoesntChangeBlock() { + cpputils::Data randomData = cpputils::DataFixture::generate(size); + auto block = CreateBlockAndLoadIt(); + WriteDataToBlock(block.get(), randomData); + block->flush(); + + EXPECT_BLOCK_DATA_CORRECT(*block, randomData); + } + + void TestAfterCreate_FlushesWhenDestructed() { + cpputils::Data randomData = cpputils::DataFixture::generate(size); + blockstore::Key key = blockstore::Key::Null(); + { + auto block = blockStore->create(cpputils::Data(size)); + key = block->key(); + WriteDataToBlock(block.get(), randomData); + } + auto loaded_block = blockStore->load(key).value(); + EXPECT_BLOCK_DATA_CORRECT(*loaded_block, randomData); + } + + void TestAfterLoad_FlushesWhenDestructed() { + cpputils::Data randomData = cpputils::DataFixture::generate(size); + blockstore::Key key = blockstore::Key::Null(); + { + key = CreateBlock()->key(); + auto block = blockStore->load(key).value(); + WriteDataToBlock(block.get(), randomData); + } + auto loaded_block = blockStore->load(key).value(); + EXPECT_BLOCK_DATA_CORRECT(*loaded_block, randomData); + } + + void TestLoadNonExistingBlock() { + EXPECT_EQ(boost::none, blockStore->load(key)); + } + +private: + const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972"); + cpputils::unique_ref blockStore; + size_t size; + + cpputils::Data ZEROES(size_t size) { + cpputils::Data ZEROES(size); + ZEROES.FillWithZeroes(); + return ZEROES; + } + + cpputils::unique_ref StoreDataToBlockAndLoadIt(const cpputils::Data &data) { + blockstore::Key key = StoreDataToBlockAndGetKey(data); + return blockStore->load(key).value(); + } + + blockstore::Key StoreDataToBlockAndGetKey(const cpputils::Data &data) { + return blockStore->create(data)->key(); + } + + cpputils::unique_ref StoreDataToBlockAndLoadItDirectlyAfterFlushing(const cpputils::Data &data) { + auto block = blockStore->create(data); + block->flush(); + return blockStore->load(block->key()).value(); + } + + cpputils::unique_ref CreateBlockAndLoadIt() { + blockstore::Key key = CreateBlock()->key(); + return blockStore->load(key).value(); + } + + cpputils::unique_ref CreateBlock() { + return blockStore->create(cpputils::Data(size)); + } + + void WriteDataToBlock(blockstore::Block *block, const cpputils::Data &randomData) { + block->write(randomData.data(), 0, randomData.size()); + } + + void EXPECT_BLOCK_DATA_CORRECT(const blockstore::Block &block, const cpputils::Data &randomData) { + EXPECT_EQ(randomData.size(), block.size()); + EXPECT_EQ(0, std::memcmp(randomData.data(), block.data(), randomData.size())); + } +}; + +constexpr std::initializer_list SIZES = {0, 1, 1024, 4096, 10*1024*1024}; +#define TYPED_TEST_P_FOR_ALL_SIZES(TestName) \ + TYPED_TEST_P(BlockStoreTest, TestName) { \ + for (auto size: SIZES) { \ + BlockStoreSizeParameterizedTest(this->fixture.createBlockStore(), size) \ + .Test##TestName(); \ + } \ + } \ + +TYPED_TEST_P_FOR_ALL_SIZES(CreatedBlockHasCorrectSize); +TYPED_TEST_P_FOR_ALL_SIZES(LoadingUnchangedBlockHasCorrectSize); +TYPED_TEST_P_FOR_ALL_SIZES(CreatedBlockData); +TYPED_TEST_P_FOR_ALL_SIZES(LoadingUnchangedBlockData); +TYPED_TEST_P_FOR_ALL_SIZES(LoadedBlockIsCorrect); +//TYPED_TEST_P_FOR_ALL_SIZES(LoadedBlockIsCorrectWhenLoadedDirectlyAfterFlushing); +TYPED_TEST_P_FOR_ALL_SIZES(AfterCreate_FlushingDoesntChangeBlock); +TYPED_TEST_P_FOR_ALL_SIZES(AfterLoad_FlushingDoesntChangeBlock); +TYPED_TEST_P_FOR_ALL_SIZES(AfterCreate_FlushesWhenDestructed); +TYPED_TEST_P_FOR_ALL_SIZES(AfterLoad_FlushesWhenDestructed); +TYPED_TEST_P_FOR_ALL_SIZES(LoadNonExistingBlock); + +#endif diff --git a/test/blockstore/testutils/BlockStoreWithRandomKeysTest.h b/test/blockstore/testutils/BlockStoreWithRandomKeysTest.h new file mode 100644 index 00000000..6dbcbf96 --- /dev/null +++ b/test/blockstore/testutils/BlockStoreWithRandomKeysTest.h @@ -0,0 +1,88 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTOREWITHRANDOMKEYSTEST_H_ +#define MESSMER_BLOCKSTORE_TEST_TESTUTILS_BLOCKSTOREWITHRANDOMKEYSTEST_H_ + +#include + +#include "../../interface/BlockStore.h" + +class BlockStoreWithRandomKeysTestFixture { +public: + virtual ~BlockStoreWithRandomKeysTestFixture() {} + virtual cpputils::unique_ref createBlockStore() = 0; +}; + +template +class BlockStoreWithRandomKeysTest: public ::testing::Test { +public: + BlockStoreWithRandomKeysTest(): fixture() {} + + BOOST_STATIC_ASSERT_MSG( + (std::is_base_of::value), + "Given test fixture for instantiating the (type parameterized) BlockStoreWithRandomKeysTest must inherit from BlockStoreWithRandomKeysTestFixture" + ); + + blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972"); + + const std::vector SIZES = {0, 1, 1024, 4096, 10*1024*1024}; + + ConcreteBlockStoreWithRandomKeysTestFixture fixture; +}; + +TYPED_TEST_CASE_P(BlockStoreWithRandomKeysTest); + +TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndSameSize) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->tryCreate(this->key, cpputils::Data(1024)); + (*block)->flush(); //TODO Ideally, flush shouldn't be necessary here. + auto block2 = blockStore->tryCreate(this->key, cpputils::Data(1024)); + EXPECT_NE(boost::none, block); + EXPECT_EQ(boost::none, block2); +} + +TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndDifferentSize) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->tryCreate(this->key, cpputils::Data(1024)); + (*block)->flush(); //TODO Ideally, flush shouldn't be necessary here. + auto block2 = blockStore->tryCreate(this->key, cpputils::Data(4096)); + EXPECT_NE(boost::none, block); + EXPECT_EQ(boost::none, block2); +} + +TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndFirstNullSize) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->tryCreate(this->key, cpputils::Data(0)); + (*block)->flush(); //TODO Ideally, flush shouldn't be necessary here. + auto block2 = blockStore->tryCreate(this->key, cpputils::Data(1024)); + EXPECT_NE(boost::none, block); + EXPECT_EQ(boost::none, block2); +} + +TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndSecondNullSize) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->tryCreate(this->key, cpputils::Data(1024)); + (*block)->flush(); //TODO Ideally, flush shouldn't be necessary here. + auto block2 = blockStore->tryCreate(this->key, cpputils::Data(0)); + EXPECT_NE(boost::none, block); + EXPECT_EQ(boost::none, block2); +} + +TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndBothNullSize) { + auto blockStore = this->fixture.createBlockStore(); + auto block = blockStore->tryCreate(this->key, cpputils::Data(0)); + (*block)->flush(); //TODO Ideally, flush shouldn't be necessary here. + auto block2 = blockStore->tryCreate(this->key, cpputils::Data(0)); + EXPECT_NE(boost::none, block); + EXPECT_EQ(boost::none, block2); +} + +REGISTER_TYPED_TEST_CASE_P(BlockStoreWithRandomKeysTest, + CreateTwoBlocksWithSameKeyAndSameSize, + CreateTwoBlocksWithSameKeyAndDifferentSize, + CreateTwoBlocksWithSameKeyAndFirstNullSize, + CreateTwoBlocksWithSameKeyAndSecondNullSize, + CreateTwoBlocksWithSameKeyAndBothNullSize +); + + +#endif diff --git a/test/blockstore/utils/BlockStoreUtilsTest.cpp b/test/blockstore/utils/BlockStoreUtilsTest.cpp new file mode 100644 index 00000000..ecd5c610 --- /dev/null +++ b/test/blockstore/utils/BlockStoreUtilsTest.cpp @@ -0,0 +1,113 @@ +#include "../../implementations/testfake/FakeBlockStore.h" +#include +#include "../../utils/BlockStoreUtils.h" +#include "google/gtest/gtest.h" + +#include + +using ::testing::Test; +using ::testing::WithParamInterface; +using ::testing::Values; + +using cpputils::Data; +using cpputils::DataFixture; +using cpputils::unique_ref; +using cpputils::make_unique_ref; + +using namespace blockstore; +using namespace blockstore::utils; + +using blockstore::testfake::FakeBlockStore; + +class BlockStoreUtilsTest: public Test { +public: + unsigned int SIZE = 1024 * 1024; + BlockStoreUtilsTest(): + ZEROES(SIZE), + dataFixture(DataFixture::generate(SIZE)), + blockStore(make_unique_ref()) { + ZEROES.FillWithZeroes(); + } + + Data ZEROES; + Data dataFixture; + unique_ref blockStore; +}; + +TEST_F(BlockStoreUtilsTest, FillWithZeroes) { + auto block = blockStore->create(Data(SIZE)); + block->write(dataFixture.data(), 0, SIZE); + EXPECT_NE(0, std::memcmp(ZEROES.data(), block->data(), SIZE)); + fillWithZeroes(block.get()); + EXPECT_EQ(0, std::memcmp(ZEROES.data(), block->data(), SIZE)); +} + +class BlockStoreUtilsTest_CopyToNewBlock: public BlockStoreUtilsTest {}; + +TEST_F(BlockStoreUtilsTest_CopyToNewBlock, CopyEmptyBlock) { + auto block = blockStore->create(Data(0)); + auto block2 = copyToNewBlock(blockStore.get(), *block); + + EXPECT_EQ(0u, block2->size()); +} + +TEST_F(BlockStoreUtilsTest_CopyToNewBlock, CopyZeroBlock) { + auto block = blockStore->create(ZEROES); + auto block2 = copyToNewBlock(blockStore.get(), *block); + + EXPECT_EQ(SIZE, block2->size()); + EXPECT_EQ(0, std::memcmp(ZEROES.data(), block2->data(), SIZE)); +} + +TEST_F(BlockStoreUtilsTest_CopyToNewBlock, CopyDataBlock) { + auto block = blockStore->create(Data(SIZE)); + block->write(dataFixture.data(), 0, SIZE); + auto block2 = copyToNewBlock(blockStore.get(), *block); + + EXPECT_EQ(SIZE, block2->size()); + EXPECT_EQ(0, std::memcmp(dataFixture.data(), block2->data(), SIZE)); +} + +TEST_F(BlockStoreUtilsTest_CopyToNewBlock, OriginalBlockUnchanged) { + auto block = blockStore->create(Data(SIZE)); + block->write(dataFixture.data(), 0, SIZE); + auto block2 = copyToNewBlock(blockStore.get(), *block); + + EXPECT_EQ(SIZE, block->size()); + EXPECT_EQ(0, std::memcmp(dataFixture.data(), block->data(), SIZE)); +} + +class BlockStoreUtilsTest_CopyToExistingBlock: public BlockStoreUtilsTest {}; + +TEST_F(BlockStoreUtilsTest_CopyToExistingBlock, CopyEmptyBlock) { + auto block = blockStore->create(Data(0)); + auto block2 = blockStore->create(Data(0)); + copyTo(block2.get(), *block); +} + +TEST_F(BlockStoreUtilsTest_CopyToExistingBlock, CopyZeroBlock) { + auto block = blockStore->create(ZEROES); + auto block2 = blockStore->create(Data(SIZE)); + block2->write(dataFixture.data(), 0, SIZE); + copyTo(block2.get(), *block); + + EXPECT_EQ(0, std::memcmp(ZEROES.data(), block2->data(), SIZE)); +} + +TEST_F(BlockStoreUtilsTest_CopyToExistingBlock, CopyDataBlock) { + auto block = blockStore->create(Data(SIZE)); + block->write(dataFixture.data(), 0, SIZE); + auto block2 = blockStore->create(Data(SIZE)); + copyTo(block2.get(), *block); + + EXPECT_EQ(0, std::memcmp(dataFixture.data(), block2->data(), SIZE)); +} + +TEST_F(BlockStoreUtilsTest_CopyToExistingBlock, OriginalBlockUnchanged) { + auto block = blockStore->create(Data(SIZE)); + block->write(dataFixture.data(), 0, SIZE); + auto block2 = blockStore->create(Data(SIZE)); + copyTo(block2.get(), *block); + + EXPECT_EQ(0, std::memcmp(dataFixture.data(), block->data(), SIZE)); +}