hardcoresushi/libcryfs
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Remove old blockstore implementations

Browse Source
This commit is contained in:
Sebastian Messmer 2017-07-20 19:32:42 -07:00
parent 36c29f19cf
commit 4a602ce7a5
46 changed files with 319 additions and 1632 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/blockstore/CMakeLists.txt
View File

@ -7,8 +7,6 @@ set(SOURCES
interface/helpers/BlockStoreWithRandomKeys.cpp
implementations/testfake/FakeBlockStore.cpp
implementations/testfake/FakeBlock.cpp
implementations/inmemory/InMemoryBlock.cpp
implementations/inmemory/InMemoryBlockStore.cpp
implementations/inmemory/InMemoryBlockStore2.cpp
implementations/parallelaccess/ParallelAccessBlockStore.cpp
implementations/parallelaccess/BlockRef.cpp
@ -17,12 +15,8 @@ set(SOURCES
implementations/compressing/CompressedBlock.cpp
implementations/compressing/compressors/RunLengthEncoding.cpp
implementations/compressing/compressors/Gzip.cpp
implementations/encrypted/EncryptedBlockStore.cpp
implementations/encrypted/EncryptedBlockStore2.cpp
implementations/encrypted/EncryptedBlock.cpp
implementations/ondisk/OnDiskBlockStore.cpp
implementations/ondisk/OnDiskBlockStore2.cpp
implementations/ondisk/OnDiskBlock.cpp
implementations/caching/CachingBlockStore.cpp
implementations/caching/cache/PeriodicTask.cpp
implementations/caching/cache/CacheEntry.cpp
@ -32,8 +26,6 @@ set(SOURCES
implementations/caching/NewBlock.cpp
implementations/low2highlevel/LowToHighLevelBlock.cpp
implementations/low2highlevel/LowToHighLevelBlockStore.cpp
implementations/versioncounting/VersionCountingBlock.cpp
implementations/versioncounting/VersionCountingBlockStore.cpp
implementations/versioncounting/VersionCountingBlockStore2.cpp
implementations/versioncounting/KnownBlockVersions.cpp
implementations/versioncounting/ClientIdAndBlockKey.cpp

1
src/blockstore/implementations/encrypted/EncryptedBlock.cpp
View File

@ -1 +0,0 @@
#include "EncryptedBlock.h"

229
src/blockstore/implementations/encrypted/EncryptedBlock.h
View File

@ -1,229 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include "../../interface/BlockStore.h"
#include <cpp-utils/macros.h>
#include <memory>
#include <iostream>
#include <boost/optional.hpp>
#include <cpp-utils/crypto/symmetric/Cipher.h>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
#include <mutex>
#include <cpp-utils/logging/logging.h>
namespace blockstore {
namespace encrypted {
template<class Cipher> class EncryptedBlockStore;
//TODO Test EncryptedBlock
//TODO Fix mutexes & locks (basically true for all blockstores)
template<class Cipher>
class EncryptedBlock final: public Block {
public:
BOOST_CONCEPT_ASSERT((cpputils::CipherConcept<Cipher>));
static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey);
static cpputils::unique_ref<EncryptedBlock> Overwrite(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey);
static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key);
static uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize);
//TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key, cpputils::Data plaintextWithHeader);
~EncryptedBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t count) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref<Block> releaseBlock();
private:
cpputils::unique_ref<Block> _baseBlock; // TODO Do I need the ciphertext block in memory or is the key enough?
cpputils::Data _plaintextWithHeader;
typename Cipher::EncryptionKey _encKey;
bool _dataChanged;
static constexpr unsigned int HEADER_LENGTH = Key::BINARY_LENGTH;
void _encryptToBaseBlock();
static cpputils::Data _prependKeyHeaderToData(const Key &key, cpputils::Data data);
static bool _keyHeaderIsCorrect(const Key &key, const cpputils::Data &data);
static cpputils::Data _prependFormatHeader(const cpputils::Data &data);
static void _checkFormatHeader(const void *data);
// This header is prepended to blocks to allow future versions to have compatibility.
static constexpr uint16_t FORMAT_VERSION_HEADER = 0;
std::mutex _mutex;
DISALLOW_COPY_AND_ASSIGN(EncryptedBlock);
};
template<class Cipher>
constexpr unsigned int EncryptedBlock<Cipher>::HEADER_LENGTH;
template<class Cipher>
constexpr uint16_t EncryptedBlock<Cipher>::FORMAT_VERSION_HEADER;
template<class Cipher>
boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey) {
//TODO Is it possible to avoid copying the whole plaintext data into plaintextWithHeader? Maybe an encrypt() object that has an .addData() function and concatenates all data for encryption? Maybe Crypto++ offers this functionality already.
cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, std::move(data));
cpputils::Data encrypted = Cipher::encrypt((byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), encKey);
//TODO Avoid copying the whole encrypted block into a encryptedWithFormatHeader by creating a Data object with full size and then giving it as an encryption target to Cipher::encrypt()
cpputils::Data encryptedWithFormatHeader = _prependFormatHeader(std::move(encrypted));
auto baseBlock = baseBlockStore->tryCreate(key, std::move(encryptedWithFormatHeader));
if (baseBlock == boost::none) {
//TODO Test this code branch
return boost::none;
}
return cpputils::make_unique_ref<EncryptedBlock>(std::move(*baseBlock), encKey, std::move(plaintextWithHeader));
}
template<class Cipher>
cpputils::unique_ref<EncryptedBlock<Cipher>> EncryptedBlock<Cipher>::Overwrite(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey) {
//TODO Is it possible to avoid copying the whole plaintext data into plaintextWithHeader? Maybe an encrypt() object that has an .addData() function and concatenates all data for encryption? Maybe Crypto++ offers this functionality already.
cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, std::move(data));
cpputils::Data encrypted = Cipher::encrypt((byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), encKey);
//TODO Avoid copying the whole encrypted block into a encryptedWithFormatHeader by creating a Data object with full size and then giving it as an encryption target to Cipher::encrypt()
cpputils::Data encryptedWithFormatHeader = _prependFormatHeader(std::move(encrypted));
auto baseBlock = baseBlockStore->overwrite(key, std::move(encryptedWithFormatHeader));
return cpputils::make_unique_ref<EncryptedBlock>(std::move(baseBlock), encKey, std::move(plaintextWithHeader));
}
template<class Cipher>
cpputils::Data EncryptedBlock<Cipher>::_prependFormatHeader(const cpputils::Data &data) {
cpputils::Data dataWithHeader(sizeof(FORMAT_VERSION_HEADER) + data.size());
std::memcpy(dataWithHeader.dataOffset(0), &FORMAT_VERSION_HEADER, sizeof(FORMAT_VERSION_HEADER));
std::memcpy(dataWithHeader.dataOffset(sizeof(FORMAT_VERSION_HEADER)), data.data(), data.size());
return dataWithHeader;
}
template<class Cipher>
boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey) {
_checkFormatHeader(baseBlock->data());
boost::optional<cpputils::Data> plaintextWithHeader = Cipher::decrypt((byte*)baseBlock->data() + sizeof(FORMAT_VERSION_HEADER), baseBlock->size() - sizeof(FORMAT_VERSION_HEADER), encKey);
if(plaintextWithHeader == boost::none) {
//Decryption failed (e.g. an authenticated cipher detected modifications to the ciphertext)
cpputils::logging::LOG(cpputils::logging::WARN, "Decrypting block {} failed. Was the block modified by an attacker?", baseBlock->key().ToString());
return boost::none;
}
if(!_keyHeaderIsCorrect(baseBlock->key(), *plaintextWithHeader)) {
//The stored key in the block data is incorrect - an attacker might have exchanged the contents with the encrypted data from a different block
cpputils::logging::LOG(cpputils::logging::WARN, "Decrypting block {} failed due to invalid block key. Was the block modified by an attacker?", baseBlock->key().ToString());
return boost::none;
}
return cpputils::make_unique_ref<EncryptedBlock<Cipher>>(std::move(baseBlock), encKey, std::move(*plaintextWithHeader));
}
template<class Cipher>
void EncryptedBlock<Cipher>::_checkFormatHeader(const void *data) {
if (*reinterpret_cast<decltype(FORMAT_VERSION_HEADER)*>(data) != FORMAT_VERSION_HEADER) {
throw std::runtime_error("The encrypted block has the wrong format. Was it created with a newer version of CryFS?");
}
}
template<class Cipher>
cpputils::Data EncryptedBlock<Cipher>::_prependKeyHeaderToData(const Key &key, cpputils::Data data) {
static_assert(HEADER_LENGTH >= Key::BINARY_LENGTH, "Key doesn't fit into the header");
cpputils::Data result(data.size() + HEADER_LENGTH);
std::memcpy(result.data(), key.data(), Key::BINARY_LENGTH);
std::memcpy((uint8_t*)result.data() + Key::BINARY_LENGTH, data.data(), data.size());
return result;
}
template<class Cipher>
bool EncryptedBlock<Cipher>::_keyHeaderIsCorrect(const Key &key, const cpputils::Data &data) {
return 0 == std::memcmp(key.data(), data.data(), Key::BINARY_LENGTH);
}
template<class Cipher>
EncryptedBlock<Cipher>::EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey, cpputils::Data plaintextWithHeader)
:Block(baseBlock->key()),
_baseBlock(std::move(baseBlock)),
_plaintextWithHeader(std::move(plaintextWithHeader)),
_encKey(encKey),
_dataChanged(false),
_mutex() {
}
template<class Cipher>
EncryptedBlock<Cipher>::~EncryptedBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
}
template<class Cipher>
const void *EncryptedBlock<Cipher>::data() const {
return (uint8_t*)_plaintextWithHeader.data() + HEADER_LENGTH;
}
template<class Cipher>
void EncryptedBlock<Cipher>::write(const void *source, uint64_t offset, uint64_t count) {
ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition
std::memcpy((uint8_t*)_plaintextWithHeader.data()+HEADER_LENGTH+offset, source, count);
_dataChanged = true;
}
template<class Cipher>
void EncryptedBlock<Cipher>::flush() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
return _baseBlock->flush();
}
template<class Cipher>
size_t EncryptedBlock<Cipher>::size() const {
return _plaintextWithHeader.size() - HEADER_LENGTH;
}
template<class Cipher>
void EncryptedBlock<Cipher>::resize(size_t newSize) {
_plaintextWithHeader = cpputils::DataUtils::resize(std::move(_plaintextWithHeader), newSize + HEADER_LENGTH);
_dataChanged = true;
}
template<class Cipher>
void EncryptedBlock<Cipher>::_encryptToBaseBlock() {
if (_dataChanged) {
cpputils::Data encrypted = Cipher::encrypt((byte*)_plaintextWithHeader.data(), _plaintextWithHeader.size(), _encKey);
if (_baseBlock->size() != sizeof(FORMAT_VERSION_HEADER) + encrypted.size()) {
_baseBlock->resize(sizeof(FORMAT_VERSION_HEADER) + encrypted.size());
}
_baseBlock->write(&FORMAT_VERSION_HEADER, 0, sizeof(FORMAT_VERSION_HEADER));
_baseBlock->write(encrypted.data(), sizeof(FORMAT_VERSION_HEADER), encrypted.size());
_dataChanged = false;
}
}
template<class Cipher>
cpputils::unique_ref<Block> EncryptedBlock<Cipher>::releaseBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
return std::move(_baseBlock);
}
template<class Cipher>
uint64_t EncryptedBlock<Cipher>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) {
if (blockSize <= Cipher::ciphertextSize(HEADER_LENGTH) + sizeof(FORMAT_VERSION_HEADER)) {
return 0;
}
return Cipher::plaintextSize(blockSize - sizeof(FORMAT_VERSION_HEADER)) - HEADER_LENGTH;
}
}
}
#endif

1
src/blockstore/implementations/encrypted/EncryptedBlockStore.cpp
View File

@ -1 +0,0 @@
#include "EncryptedBlockStore.h"

110
src/blockstore/implementations/encrypted/EncryptedBlockStore.h
View File

@ -1,110 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
#include "../../interface/BlockStore.h"
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/cast.h>
#include "EncryptedBlock.h"
#include <iostream>
namespace blockstore {
namespace encrypted {
template<class Cipher>
class EncryptedBlockStore final: public BlockStore {
public:
EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey);
//TODO Are createKey() tests included in generic BlockStoreTest? If not, add it!
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
cpputils::unique_ref<Block> overwrite(const blockstore::Key &key, cpputils::Data data) override;
void remove(const Key &key) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const Key &)> callback) const override;
//This function should only be used by test cases
void __setKey(const typename Cipher::EncryptionKey &encKey);
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
typename Cipher::EncryptionKey _encKey;
DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore);
};
template<class Cipher>
EncryptedBlockStore<Cipher>::EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
: _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
}
template<class Cipher>
Key EncryptedBlockStore<Cipher>::createKey() {
return _baseBlockStore->createKey();
}
template<class Cipher>
boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::tryCreate(const Key &key, cpputils::Data data) {
//TODO Test that this returns boost::none when base blockstore returns nullptr (for all pass-through-blockstores)
//TODO Easier implementation? This is only so complicated because of the cast EncryptedBlock -> Block
auto result = EncryptedBlock<Cipher>::TryCreateNew(_baseBlockStore.get(), key, std::move(data), _encKey);
if (result == boost::none) {
return boost::none;
}
return cpputils::unique_ref<Block>(std::move(*result));
}
template<class Cipher>
boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::load(const Key &key) {
auto block = _baseBlockStore->load(key);
if (block == boost::none) {
//TODO Test this path (for all pass-through-blockstores)
return boost::none;
}
return boost::optional<cpputils::unique_ref<Block>>(EncryptedBlock<Cipher>::TryDecrypt(std::move(*block), _encKey));
}
template<class Cipher>
cpputils::unique_ref<Block> EncryptedBlockStore<Cipher>::overwrite(const blockstore::Key &key, cpputils::Data data) {
return EncryptedBlock<Cipher>::Overwrite(_baseBlockStore.get(), key, std::move(data), _encKey);
}
template<class Cipher>
void EncryptedBlockStore<Cipher>::remove(const Key &key) {
return _baseBlockStore->remove(key);
}
template<class Cipher>
uint64_t EncryptedBlockStore<Cipher>::numBlocks() const {
return _baseBlockStore->numBlocks();
}
template<class Cipher>
uint64_t EncryptedBlockStore<Cipher>::estimateNumFreeBytes() const {
return _baseBlockStore->estimateNumFreeBytes();
}
template<class Cipher>
void EncryptedBlockStore<Cipher>::__setKey(const typename Cipher::EncryptionKey &encKey) {
_encKey = encKey;
}
template<class Cipher>
uint64_t EncryptedBlockStore<Cipher>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return EncryptedBlock<Cipher>::blockSizeFromPhysicalBlockSize(_baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize));
}
template<class Cipher>
void EncryptedBlockStore<Cipher>::forEachBlock(std::function<void (const Key &)> callback) const {
return _baseBlockStore->forEachBlock(callback);
}
}
}
#endif

32
src/blockstore/implementations/encrypted/EncryptedBlockStore2.h
View File

@ -18,15 +18,18 @@ public:
EncryptedBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const typename Cipher::EncryptionKey &encKey);
boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override;
boost::future<bool> remove(const Key &key) override;
boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override;
boost::future<void> store(const Key &key, const cpputils::Data &data) override;
bool tryCreate(const Key &key, const cpputils::Data &data) override;
bool remove(const Key &key) override;
boost::optional<cpputils::Data> load(const Key &key) const override;
void store(const Key &key, const cpputils::Data &data) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const Key &)> callback) const override;
//This function should only be used by test cases
void __setKey(const typename Cipher::EncryptionKey &encKey);
private:
// This header is prepended to blocks to allow future versions to have compatibility.
@ -54,30 +57,28 @@ inline EncryptedBlockStore2<Cipher>::EncryptedBlockStore2(cpputils::unique_ref<B
}
template<class Cipher>
inline boost::future<bool> EncryptedBlockStore2<Cipher>::tryCreate(const Key &key, const cpputils::Data &data) {
inline bool EncryptedBlockStore2<Cipher>::tryCreate(const Key &key, const cpputils::Data &data) {
cpputils::Data encrypted = _encrypt(key, data);
return _baseBlockStore->tryCreate(key, encrypted);
}
template<class Cipher>
inline boost::future<bool> EncryptedBlockStore2<Cipher>::remove(const Key &key) {
inline bool EncryptedBlockStore2<Cipher>::remove(const Key &key) {
return _baseBlockStore->remove(key);
}
template<class Cipher>
inline boost::future<boost::optional<cpputils::Data>> EncryptedBlockStore2<Cipher>::load(const Key &key) const {
inline boost::optional<cpputils::Data> EncryptedBlockStore2<Cipher>::load(const Key &key) const {
auto loaded = _baseBlockStore->load(key);
return loaded.then([this, key] (boost::future<boost::optional<cpputils::Data>> data_) {
auto data = data_.get();
if (boost::none == data) {
if (boost::none == loaded) {
return boost::optional<cpputils::Data>(boost::none);
}
return _tryDecrypt(key, *data);
});
return _tryDecrypt(key, *loaded);
}
template<class Cipher>
inline boost::future<void> EncryptedBlockStore2<Cipher>::store(const Key &key, const cpputils::Data &data) {
inline void EncryptedBlockStore2<Cipher>::store(const Key &key, const cpputils::Data &data) {
cpputils::Data encrypted = _encrypt(key, data);
return _baseBlockStore->store(key, encrypted);
}
@ -162,6 +163,11 @@ inline cpputils::Data EncryptedBlockStore2<Cipher>::_checkAndRemoveFormatHeader(
return data.copyAndRemovePrefix(sizeof(FORMAT_VERSION_HEADER));
}
template<class Cipher>
void EncryptedBlockStore2<Cipher>::__setKey(const typename Cipher::EncryptionKey &encKey) {
_encKey = encKey;
}
}
}

55
src/blockstore/implementations/inmemory/InMemoryBlock.cpp
View File

@ -1,55 +0,0 @@
#include "InMemoryBlock.h"
#include "InMemoryBlockStore.h"
#include <cstring>
#include <cpp-utils/data/DataUtils.h>
#include <cpp-utils/assert/assert.h>
using std::make_shared;
using std::istream;
using std::ostream;
using std::ifstream;
using std::ofstream;
using std::ios;
using cpputils::Data;
namespace blockstore {
namespace inmemory {
InMemoryBlock::InMemoryBlock(const Key &key, Data data)
: Block(key), _data(make_shared<Data>(std::move(data))) {
}
InMemoryBlock::InMemoryBlock(const InMemoryBlock &rhs)
: Block(rhs), _data(rhs._data) {
}
InMemoryBlock::~InMemoryBlock() {
}
InMemoryBlock &InMemoryBlock::operator=(const InMemoryBlock &rhs) {
_data = rhs._data;
return *this;
}
const void *InMemoryBlock::data() const {
return _data->data();
}
void InMemoryBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data->size() && offset + size <= _data->size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
std::memcpy((uint8_t*)_data->data()+offset, source, size);
}
size_t InMemoryBlock::size() const {
return _data->size();
}
void InMemoryBlock::resize(size_t newSize) {
*_data = cpputils::DataUtils::resize(std::move(*_data), newSize);
}
void InMemoryBlock::flush() {
}
}
}

34
src/blockstore/implementations/inmemory/InMemoryBlock.h
View File

@ -1,34 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
namespace blockstore {
namespace inmemory {
class InMemoryBlockStore;
class InMemoryBlock final: public Block {
public:
InMemoryBlock(const Key &key, cpputils::Data size);
InMemoryBlock(const InMemoryBlock &rhs);
~InMemoryBlock();
InMemoryBlock &operator=(const InMemoryBlock &rhs);
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
private:
std::shared_ptr<cpputils::Data> _data;
};
}
}
#endif

82
src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp
View File

@ -1,82 +0,0 @@
#include "InMemoryBlock.h"
#include "InMemoryBlockStore.h"
#include <memory>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/system/get_total_memory.h>
using std::make_unique;
using std::string;
using std::mutex;
using std::lock_guard;
using std::piecewise_construct;
using std::make_tuple;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
namespace blockstore {
namespace inmemory {
InMemoryBlockStore::InMemoryBlockStore()
: _blocks() {}
optional<unique_ref<Block>> InMemoryBlockStore::tryCreate(const Key &key, Data data) {
auto insert_result = _blocks.emplace(piecewise_construct, make_tuple(key), make_tuple(key, std::move(data)));
if (!insert_result.second) {
return none;
}
//Return a pointer to the stored InMemoryBlock
return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(insert_result.first->second));
}
optional<unique_ref<Block>> InMemoryBlockStore::load(const Key &key) {
//Return a pointer to the stored InMemoryBlock
try {
return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(_blocks.at(key)));
} catch (const std::out_of_range &e) {
return none;
}
}
unique_ref<Block> InMemoryBlockStore::overwrite(const Key &key, Data data) {
InMemoryBlock newBlock(key, std::move(data));
auto insert_result = _blocks.emplace(key, newBlock);
if (!insert_result.second) {
// If block already exists, overwrite it.
insert_result.first->second = newBlock;
}
//Return a pointer to the stored InMemoryBlock
return make_unique_ref<InMemoryBlock>(insert_result.first->second);
}
void InMemoryBlockStore::remove(const Key &key) {
int numRemoved = _blocks.erase(key);
ASSERT(1==numRemoved, "Didn't find block to remove");
}
uint64_t InMemoryBlockStore::numBlocks() const {
return _blocks.size();
}
uint64_t InMemoryBlockStore::estimateNumFreeBytes() const {
return cpputils::system::get_total_memory();
}
uint64_t InMemoryBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return blockSize;
}
void InMemoryBlockStore::forEachBlock(std::function<void (const Key &)> callback) const {
for (const auto &entry : _blocks) {
callback(entry.first);
}
}
}
}

38
src/blockstore/implementations/inmemory/InMemoryBlockStore.h
View File

@ -1,38 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
#include <cpp-utils/macros.h>
#include "InMemoryBlock.h"
#include <mutex>
#include <unordered_map>
#include "InMemoryBlock.h"
namespace blockstore {
namespace inmemory {
class InMemoryBlockStore final: public BlockStoreWithRandomKeys {
public:
InMemoryBlockStore();
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
cpputils::unique_ref<Block> overwrite(const blockstore::Key &key, cpputils::Data data) override;
void remove(const Key &key) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const Key &)> callback) const override;
private:
std::unordered_map<Key, InMemoryBlock> _blocks;
DISALLOW_COPY_AND_ASSIGN(InMemoryBlockStore);
};
}
}
#endif

35
src/blockstore/implementations/inmemory/InMemoryBlockStore2.cpp
View File

@ -16,8 +16,6 @@ using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
using boost::future;
using boost::make_ready_future;
namespace blockstore {
namespace inmemory {
@ -25,46 +23,49 @@ namespace inmemory {
InMemoryBlockStore2::InMemoryBlockStore2()
: _blocks() {}
future<bool> InMemoryBlockStore2::tryCreate(const Key &key, const Data &data) {
bool InMemoryBlockStore2::tryCreate(const Key &key, const Data &data) {
std::unique_lock<std::mutex> lock(_mutex);
auto result = _blocks.insert(make_pair(key, data.copy()));
return make_ready_future(result.second); // Return if insertion was successful (i.e. key didn't exist yet)
return _tryCreate(key, data);
}
future<bool> InMemoryBlockStore2::remove(const Key &key) {
bool InMemoryBlockStore2::_tryCreate(const Key &key, const Data &data) {
auto result = _blocks.insert(make_pair(key, data.copy()));
return result.second; // Return if insertion was successful (i.e. key didn't exist yet)
}
bool InMemoryBlockStore2::remove(const Key &key) {
std::unique_lock<std::mutex> lock(_mutex);
auto found = _blocks.find(key);
if (found == _blocks.end()) {
// Key not found
return make_ready_future(false);
return false;
}
_blocks.erase(found);
return make_ready_future(true);
return true;
}
future<optional<Data>> InMemoryBlockStore2::load(const Key &key) const {
optional<Data> InMemoryBlockStore2::load(const Key &key) const {
std::unique_lock<std::mutex> lock(_mutex);
auto found = _blocks.find(key);
if (found == _blocks.end()) {
return make_ready_future(optional<Data>(none));
return boost::none;
}
return make_ready_future(optional<Data>(found->second.copy()));
return found->second.copy();
}
future<void> InMemoryBlockStore2::store(const Key &key, const Data &data) {
void InMemoryBlockStore2::store(const Key &key, const Data &data) {
std::unique_lock<std::mutex> lock(_mutex);
auto found = _blocks.find(key);
if (found == _blocks.end()) {
return tryCreate(key, data).then([] (future<bool> success) {
if (!success.get()) {
bool success = _tryCreate(key, data);
if (!success) {
throw std::runtime_error("Could neither save nor create the block in InMemoryBlockStore::store()");
}
});
}
} else {
// TODO Would have better performance: found->second.overwriteWith(data)
found->second = data.copy();
return make_ready_future();
}
}
uint64_t InMemoryBlockStore2::numBlocks() const {

9
src/blockstore/implementations/inmemory/InMemoryBlockStore2.h
View File

@ -13,10 +13,10 @@ class InMemoryBlockStore2 final: public BlockStore2 {
public:
InMemoryBlockStore2();
boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override;
boost::future<bool> remove(const Key &key) override;
boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override;
boost::future<void> store(const Key &key, const cpputils::Data &data) override;
bool tryCreate(const Key &key, const cpputils::Data &data) override;
bool remove(const Key &key) override;
boost::optional<cpputils::Data> load(const Key &key) const override;
void store(const Key &key, const cpputils::Data &data) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
@ -24,6 +24,7 @@ public:
private:
std::vector<Key> _allBlockKeys() const;
bool _tryCreate(const Key &key, const cpputils::Data &data);
std::unordered_map<Key, cpputils::Data> _blocks;
mutable std::mutex _mutex;

10
src/blockstore/implementations/low2highlevel/LowToHighLevelBlock.cpp
View File

@ -13,8 +13,7 @@ namespace blockstore {
namespace lowtohighlevel {
optional<unique_ref<LowToHighLevelBlock>> LowToHighLevelBlock::TryCreateNew(BlockStore2 *baseBlockStore, const Key &key, Data data) {
// TODO .get() is blocking
bool success = baseBlockStore->tryCreate(key, data.copy()).get(); // TODO Copy necessary?
bool success = baseBlockStore->tryCreate(key, data);
if (!success) {
return none;
}
@ -23,13 +22,12 @@ optional<unique_ref<LowToHighLevelBlock>> LowToHighLevelBlock::TryCreateNew(Bloc
}
unique_ref<LowToHighLevelBlock> LowToHighLevelBlock::Overwrite(BlockStore2 *baseBlockStore, const Key &key, Data data) {
// TODO This is blocking
baseBlockStore->store(key, data).wait(); // TODO Does it make sense to not store here, but only write back in the destructor of LowToHighLevelBlock? Also: What about tryCreate?
baseBlockStore->store(key, data); // TODO Does it make sense to not store here, but only write back in the destructor of LowToHighLevelBlock? Also: What about tryCreate?
return make_unique_ref<LowToHighLevelBlock>(key, std::move(data), baseBlockStore);
}
optional<unique_ref<LowToHighLevelBlock>> LowToHighLevelBlock::Load(BlockStore2 *baseBlockStore, const Key &key) {
optional<Data> loadedData = baseBlockStore->load(key).get(); // TODO .get() is blocking
optional<Data> loadedData = baseBlockStore->load(key);
if (loadedData == none) {
return none;
}
@ -75,7 +73,7 @@ void LowToHighLevelBlock::resize(size_t newSize) {
void LowToHighLevelBlock::_storeToBaseBlock() {
if (_dataChanged) {
_baseBlockStore->store(key(), _data).wait(); // TODO This is blocking
_baseBlockStore->store(key(), _data);
_dataChanged = false;
}
}

5
src/blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.cpp
View File

@ -46,7 +46,10 @@ optional<unique_ref<Block>> LowToHighLevelBlockStore::load(const Key &key) {
}
void LowToHighLevelBlockStore::remove(const Key &key) {
_baseBlockStore->remove(key);
bool success = _baseBlockStore->remove(key);
if (!success) {
throw std::runtime_error("Couldn't delete block with id " + key.ToString());
}
}
uint64_t LowToHighLevelBlockStore::numBlocks() const {

165
src/blockstore/implementations/ondisk/OnDiskBlock.cpp
View File

@ -1,165 +0,0 @@
#include <cstring>
#include <fstream>
#include <boost/filesystem.hpp>
#include "OnDiskBlock.h"
#include "OnDiskBlockStore.h"
#include "../../utils/FileDoesntExistException.h"
#include <cpp-utils/data/DataUtils.h>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/logging/logging.h>
using std::istream;
using std::ostream;
using std::ifstream;
using std::ofstream;
using std::ios;
using std::string;
using cpputils::Data;
using cpputils::make_unique_ref;
using cpputils::unique_ref;
using boost::optional;
using boost::none;
namespace bf = boost::filesystem;
using namespace cpputils::logging;
namespace blockstore {
namespace ondisk {
const string OnDiskBlock::FORMAT_VERSION_HEADER_PREFIX = "cryfs;block;";
const string OnDiskBlock::FORMAT_VERSION_HEADER = OnDiskBlock::FORMAT_VERSION_HEADER_PREFIX + "0";
OnDiskBlock::OnDiskBlock(const Key &key, const bf::path &filepath, Data data)
: Block(key), _filepath(filepath), _data(std::move(data)), _dataChanged(false), _mutex() {
}
OnDiskBlock::~OnDiskBlock() {
flush();
}
const void *OnDiskBlock::data() const {
return _data.data();
}
void OnDiskBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
std::memcpy(_data.dataOffset(offset), source, size);
_dataChanged = true;
}
size_t OnDiskBlock::size() const {
return _data.size();
}
void OnDiskBlock::resize(size_t newSize) {
_data = cpputils::DataUtils::resize(std::move(_data), newSize);
_dataChanged = true;
}
bf::path OnDiskBlock::_getFilepath(const bf::path &rootdir, const Key &key) {
string keyStr = key.ToString();
return rootdir / keyStr.substr(0,3) / keyStr.substr(3);
}
optional<unique_ref<OnDiskBlock>> OnDiskBlock::LoadFromDisk(const bf::path &rootdir, const Key &key) {
auto filepath = _getFilepath(rootdir, key);
try {
boost::optional<Data> data = _loadFromDisk(filepath);
if (data == none) {
return none;
}
return make_unique_ref<OnDiskBlock>(key, filepath, std::move(*data));
} catch (const FileDoesntExistException &e) {
return none;
}
}
optional<unique_ref<OnDiskBlock>> OnDiskBlock::CreateOnDisk(const bf::path &rootdir, const Key &key, Data data) {
auto filepath = _getFilepath(rootdir, key);
bf::create_directory(filepath.parent_path());
if (bf::exists(filepath)) {
return none;
}
auto block = make_unique_ref<OnDiskBlock>(key, filepath, std::move(data));
block->_storeToDisk();
return std::move(block);
}
unique_ref<OnDiskBlock> OnDiskBlock::OverwriteOnDisk(const bf::path &rootdir, const Key &key, Data data) {
auto filepath = _getFilepath(rootdir, key);
bf::create_directory(filepath.parent_path());
auto block = make_unique_ref<OnDiskBlock>(key, filepath, std::move(data));
block->_storeToDisk();
return std::move(block);
}
void OnDiskBlock::RemoveFromDisk(const bf::path &rootdir, const Key &key) {
auto filepath = _getFilepath(rootdir, key);
ASSERT(bf::is_regular_file(filepath), "Block not found on disk");
bool retval = bf::remove(filepath);
if (!retval) {
LOG(ERROR, "Couldn't find block {} to remove", key.ToString());
}
if (bf::is_empty(filepath.parent_path())) {
bf::remove(filepath.parent_path());
}
}
void OnDiskBlock::_storeToDisk() const {
Data fileContent(formatVersionHeaderSize() + _data.size());
std::memcpy(fileContent.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
std::memcpy(fileContent.dataOffset(formatVersionHeaderSize()), _data.data(), _data.size());
fileContent.StoreToFile(_filepath);
}
optional<Data> OnDiskBlock::_loadFromDisk(const bf::path &filepath) {
auto fileContent = Data::LoadFromFile(filepath);
if (fileContent == none) {
return none;
}
return _checkAndRemoveHeader(std::move(*fileContent));
}
Data OnDiskBlock::_checkAndRemoveHeader(Data data) {
if (!_isAcceptedCryfsHeader(data)) {
if (_isOtherCryfsHeader(data)) {
throw std::runtime_error("This block is not supported yet. Maybe it was created with a newer version of CryFS?");
} else {
throw std::runtime_error("This is not a valid block.");
}
}
Data result(data.size() - formatVersionHeaderSize());
std::memcpy(result.data(), data.dataOffset(formatVersionHeaderSize()), result.size());
return result;
}
bool OnDiskBlock::_isAcceptedCryfsHeader(const Data &data) {
return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
}
bool OnDiskBlock::_isOtherCryfsHeader(const Data &data) {
return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER_PREFIX.c_str(), FORMAT_VERSION_HEADER_PREFIX.size());
}
unsigned int OnDiskBlock::formatVersionHeaderSize() {
return FORMAT_VERSION_HEADER.size() + 1; // +1 because of the null byte
}
void OnDiskBlock::flush() {
std::unique_lock<std::mutex> lock(_mutex);
if (_dataChanged) {
_storeToDisk();
_dataChanged = false;
}
}
uint64_t OnDiskBlock::blockSizeFromPhysicalBlockSize(uint64_t blockSize) {
if(blockSize <= formatVersionHeaderSize()) {
return 0;
}
return blockSize - formatVersionHeaderSize();
}
}
}

62
src/blockstore/implementations/ondisk/OnDiskBlock.h
View File

@ -1,62 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
#include <boost/filesystem/path.hpp>
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include <iostream>
#include <cpp-utils/pointer/unique_ref.h>
#include <mutex>
namespace blockstore {
namespace ondisk {
class OnDiskBlockStore;
class OnDiskBlock final: public Block {
public:
OnDiskBlock(const Key &key, const boost::filesystem::path &filepath, cpputils::Data data);
~OnDiskBlock();
static const std::string FORMAT_VERSION_HEADER_PREFIX;
static const std::string FORMAT_VERSION_HEADER;
static unsigned int formatVersionHeaderSize();
static uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize);
static boost::optional<cpputils::unique_ref<OnDiskBlock>> LoadFromDisk(const boost::filesystem::path &rootdir, const Key &key);
static boost::optional<cpputils::unique_ref<OnDiskBlock>> CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data);
static cpputils::unique_ref<OnDiskBlock> OverwriteOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data);
static void RemoveFromDisk(const boost::filesystem::path &rootdir, const Key &key);
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
private:
static bool _isAcceptedCryfsHeader(const cpputils::Data &data);
static bool _isOtherCryfsHeader(const cpputils::Data &data);
static cpputils::Data _checkAndRemoveHeader(cpputils::Data data);
static boost::filesystem::path _getFilepath(const boost::filesystem::path &rootdir, const Key &key);
const boost::filesystem::path _filepath;
cpputils::Data _data;
bool _dataChanged;
static boost::optional<cpputils::Data> _loadFromDisk(const boost::filesystem::path &filepath);
void _storeToDisk() const;
std::mutex _mutex;
DISALLOW_COPY_AND_ASSIGN(OnDiskBlock);
};
}
}
#endif

113
src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp
View File

@ -1,113 +0,0 @@
#include "OnDiskBlock.h"
#include "OnDiskBlockStore.h"
#include <sys/statvfs.h>
using std::string;
using cpputils::Data;
using cpputils::unique_ref;
using boost::optional;
using boost::none;
using std::vector;
namespace bf = boost::filesystem;
namespace blockstore {
namespace ondisk {
OnDiskBlockStore::OnDiskBlockStore(const boost::filesystem::path &rootdir)
: _rootdir(rootdir) {
if (!bf::exists(rootdir)) {
throw std::runtime_error("Base directory not found");
}
if (!bf::is_directory(rootdir)) {
throw std::runtime_error("Base directory is not a directory");
}
//TODO Test for read access, write access, enter (x) access, and throw runtime_error in case
#ifndef CRYFS_NO_COMPATIBILITY
_migrateBlockStore();
#endif
}
#ifndef CRYFS_NO_COMPATIBILITY
void OnDiskBlockStore::_migrateBlockStore() {
vector<string> blocksToMigrate;
for (auto entry = bf::directory_iterator(_rootdir); entry != bf::directory_iterator(); ++entry) {
if (bf::is_regular_file(entry->path()) && _isValidBlockKey(entry->path().filename().native())) {
blocksToMigrate.push_back(entry->path().filename().native());
}
}
if (blocksToMigrate.size() != 0) {
std::cout << "Migrating CryFS filesystem..." << std::flush;
for (auto key : blocksToMigrate) {
Key::FromString(key); // Assert that it can be parsed as a key
string dir = key.substr(0, 3);
string file = key.substr(3);
bf::create_directory(_rootdir / dir);
bf::rename(_rootdir / key, _rootdir / dir / file);
}
std::cout << "done" << std::endl;
}
}
bool OnDiskBlockStore::_isValidBlockKey(const string &key) {
return key.size() == 32 && key.find_first_not_of("0123456789ABCDEF") == string::npos;
}
#endif
//TODO Do I have to lock tryCreate/remove and/or load? Or does ParallelAccessBlockStore take care of that?
optional<unique_ref<Block>> OnDiskBlockStore::tryCreate(const Key &key, Data data) {
//TODO Easier implementation? This is only so complicated because of the cast OnDiskBlock -> Block
auto result = std::move(OnDiskBlock::CreateOnDisk(_rootdir, key, std::move(data)));
if (result == boost::none) {
return boost::none;
}
return unique_ref<Block>(std::move(*result));
}
unique_ref<Block> OnDiskBlockStore::overwrite(const Key &key, Data data) {
return OnDiskBlock::OverwriteOnDisk(_rootdir, key, std::move(data));
}
optional<unique_ref<Block>> OnDiskBlockStore::load(const Key &key) {
return optional<unique_ref<Block>>(OnDiskBlock::LoadFromDisk(_rootdir, key));
}
void OnDiskBlockStore::remove(const Key &key) {
OnDiskBlock::RemoveFromDisk(_rootdir, key);
}
uint64_t OnDiskBlockStore::numBlocks() const {
uint64_t count = 0;
for (auto prefixDir = bf::directory_iterator(_rootdir); prefixDir != bf::directory_iterator(); ++prefixDir) {
if (bf::is_directory(prefixDir->path())) {
count += std::distance(bf::directory_iterator(prefixDir->path()), bf::directory_iterator());
}
}
return count;
}
uint64_t OnDiskBlockStore::estimateNumFreeBytes() const {
struct statvfs stat;
::statvfs(_rootdir.c_str(), &stat);
return stat.f_bsize*stat.f_bavail;
}
uint64_t OnDiskBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return OnDiskBlock::blockSizeFromPhysicalBlockSize(blockSize);
}
void OnDiskBlockStore::forEachBlock(std::function<void (const Key &)> callback) const {
for (auto prefixDir = bf::directory_iterator(_rootdir); prefixDir != bf::directory_iterator(); ++prefixDir) {
if (bf::is_directory(prefixDir->path())) {
string blockKeyPrefix = prefixDir->path().filename().native();
for (auto block = bf::directory_iterator(prefixDir->path()); block != bf::directory_iterator(); ++block) {
string blockKeyPostfix = block->path().filename().native();
callback(Key::FromString(blockKeyPrefix + blockKeyPostfix));
}
}
}
}
}
}

40
src/blockstore/implementations/ondisk/OnDiskBlockStore.h
View File

@ -1,40 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
#include <boost/filesystem.hpp>
#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
#include "../../interface/BlockStore2.h"
#include <cpp-utils/macros.h>
namespace blockstore {
namespace ondisk {
class OnDiskBlockStore final: public BlockStoreWithRandomKeys {
public:
explicit OnDiskBlockStore(const boost::filesystem::path &rootdir);
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
//TODO Can we make this faster by allowing to delete blocks by only having their Key? So we wouldn't have to load it first?
void remove(const Key &key) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const Key &)> callback) const override;
cpputils::unique_ref<Block> overwrite(const blockstore::Key &key, cpputils::Data data) override;
private:
const boost::filesystem::path _rootdir;
#ifndef CRYFS_NO_COMPATIBILITY
void _migrateBlockStore();
bool _isValidBlockKey(const std::string &key);
#endif
DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore);
};
}
}
#endif

27
src/blockstore/implementations/ondisk/OnDiskBlockStore2.cpp
View File

@ -1,7 +1,7 @@
#include "OnDiskBlockStore2.h"
#include <boost/filesystem.hpp>
using std::string;
using boost::future;
using boost::optional;
using boost::none;
using cpputils::Data;
@ -45,48 +45,47 @@ unsigned int OnDiskBlockStore2::formatVersionHeaderSize() {
OnDiskBlockStore2::OnDiskBlockStore2(const boost::filesystem::path& path)
: _rootDir(path) {}
future<bool> OnDiskBlockStore2::tryCreate(const Key &key, const Data &data) {
bool OnDiskBlockStore2::tryCreate(const Key &key, const Data &data) {
auto filepath = _getFilepath(key);
if (boost::filesystem::exists(filepath)) {
return boost::make_ready_future(false);
return false;
}
store(key, data).wait();
return boost::make_ready_future(true);
store(key, data);
return true;
}
future<bool> OnDiskBlockStore2::remove(const Key &key) {
bool OnDiskBlockStore2::remove(const Key &key) {
auto filepath = _getFilepath(key);
if (!boost::filesystem::is_regular_file(filepath)) { // TODO Is this branch necessary?
return boost::make_ready_future(false);
return false;
}
bool retval = boost::filesystem::remove(filepath);
if (!retval) {
cpputils::logging::LOG(cpputils::logging::ERROR, "Couldn't find block {} to remove", key.ToString());
return boost::make_ready_future(false);
return false;
}
if (boost::filesystem::is_empty(filepath.parent_path())) {
boost::filesystem::remove(filepath.parent_path());
}
return boost::make_ready_future(true);
return true;
}
future<optional<Data>> OnDiskBlockStore2::load(const Key &key) const {
optional<Data> OnDiskBlockStore2::load(const Key &key) const {
auto fileContent = Data::LoadFromFile(_getFilepath(key));
if (fileContent == none) {
return boost::make_ready_future(optional<Data>(none));
return boost::none;
}
return boost::make_ready_future(optional<Data>(_checkAndRemoveHeader(std::move(*fileContent))));
return _checkAndRemoveHeader(std::move(*fileContent));
}
future<void> OnDiskBlockStore2::store(const Key &key, const Data &data) {
void OnDiskBlockStore2::store(const Key &key, const Data &data) {
Data fileContent(formatVersionHeaderSize() + data.size());
std::memcpy(fileContent.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
std::memcpy(fileContent.dataOffset(formatVersionHeaderSize()), data.data(), data.size());
auto filepath = _getFilepath(key);
boost::filesystem::create_directory(filepath.parent_path()); // TODO Instead create all of them once at fs creation time?
fileContent.StoreToFile(filepath);
return boost::make_ready_future();
}
uint64_t OnDiskBlockStore2::numBlocks() const {

9
src/blockstore/implementations/ondisk/OnDiskBlockStore2.h
View File

@ -6,7 +6,6 @@
#include <boost/filesystem/path.hpp>
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/unique_ref.h>
#include "OnDiskBlockStore.h"
#include <cpp-utils/logging/logging.h>
#include <sys/statvfs.h>
@ -17,10 +16,10 @@ class OnDiskBlockStore2 final: public BlockStore2 {
public:
explicit OnDiskBlockStore2(const boost::filesystem::path& path);
boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override;
boost::future<bool> remove(const Key &key) override;
boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override;
boost::future<void> store(const Key &key, const cpputils::Data &data) override;
bool tryCreate(const Key &key, const cpputils::Data &data) override;
bool remove(const Key &key) override;
boost::optional<cpputils::Data> load(const Key &key) const override;
void store(const Key &key, const cpputils::Data &data) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;

11
src/blockstore/implementations/versioncounting/VersionCountingBlock.cpp
View File

@ -1,11 +0,0 @@
#include "VersionCountingBlock.h"
namespace blockstore {
namespace versioncounting {
constexpr unsigned int VersionCountingBlock::CLIENTID_HEADER_OFFSET;
constexpr unsigned int VersionCountingBlock::VERSION_HEADER_OFFSET;
constexpr unsigned int VersionCountingBlock::HEADER_LENGTH;
constexpr uint16_t VersionCountingBlock::FORMAT_VERSION_HEADER;
constexpr uint64_t VersionCountingBlock::VERSION_ZERO;
}
}

213
src/blockstore/implementations/versioncounting/VersionCountingBlock.h
View File

@ -1,213 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_VERSIONCOUNTING_VERSIONCOUNTINGBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_VERSIONCOUNTING_VERSIONCOUNTINGBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include "../../interface/BlockStore.h"
#include "KnownBlockVersions.h"
#include <cpp-utils/macros.h>
#include <memory>
#include <iostream>
#include <boost/optional.hpp>
#include <cpp-utils/crypto/symmetric/Cipher.h>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
#include <mutex>
#include <cpp-utils/logging/logging.h>
#include "IntegrityViolationError.h"
#include "VersionCountingBlockStore.h"
namespace blockstore {
namespace versioncounting {
// TODO Is an implementation that doesn't keep an in-memory copy but just passes through write() calls to the underlying block store (including a write call to the version number each time) faster?
class VersionCountingBlock final: public Block {
public:
static boost::optional<cpputils::unique_ref<VersionCountingBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, VersionCountingBlockStore *blockStore);
static cpputils::unique_ref<VersionCountingBlock> Overwrite(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, VersionCountingBlockStore *blockStore);
static cpputils::unique_ref<VersionCountingBlock> Load(cpputils::unique_ref<Block> baseBlock, VersionCountingBlockStore *blockStore);
static uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize);
//TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
VersionCountingBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data dataWithHeader, VersionCountingBlockStore *blockStore);
~VersionCountingBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t count) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref<Block> releaseBlock();
private:
VersionCountingBlockStore *_blockStore;
cpputils::unique_ref<Block> _baseBlock;
cpputils::Data _dataWithHeader;
bool _dataChanged;
std::mutex _mutex;
void _storeToBaseBlock();
static cpputils::Data _prependHeaderToData(uint32_t myClientId, uint64_t version, cpputils::Data data);
static void _checkFormatHeader(const cpputils::Data &data);
uint64_t _readVersion();
uint32_t _readClientId();
void _checkVersion();
// This header is prepended to blocks to allow future versions to have compatibility.
static constexpr uint16_t FORMAT_VERSION_HEADER = 0;
DISALLOW_COPY_AND_ASSIGN(VersionCountingBlock);
public:
static constexpr uint64_t VERSION_ZERO = 0;
static constexpr unsigned int CLIENTID_HEADER_OFFSET = sizeof(FORMAT_VERSION_HEADER);
static constexpr unsigned int VERSION_HEADER_OFFSET = sizeof(FORMAT_VERSION_HEADER) + sizeof(uint32_t);
static constexpr unsigned int HEADER_LENGTH = sizeof(FORMAT_VERSION_HEADER) + sizeof(uint32_t) + sizeof(VERSION_ZERO);
};
inline boost::optional<cpputils::unique_ref<VersionCountingBlock>> VersionCountingBlock::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, VersionCountingBlockStore *blockStore) {
uint64_t version = blockStore->knownBlockVersions()->incrementVersion(key);
cpputils::Data dataWithHeader = _prependHeaderToData(blockStore->knownBlockVersions()->myClientId(), version, std::move(data));
auto baseBlock = baseBlockStore->tryCreate(key, dataWithHeader.copy()); // TODO Copy necessary?
if (baseBlock == boost::none) {
//TODO Test this code branch
return boost::none;
}
return cpputils::make_unique_ref<VersionCountingBlock>(std::move(*baseBlock), std::move(dataWithHeader), blockStore);
}
inline cpputils::unique_ref<VersionCountingBlock> VersionCountingBlock::Overwrite(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, VersionCountingBlockStore *blockStore) {
uint64_t version = blockStore->knownBlockVersions()->incrementVersion(key);
cpputils::Data dataWithHeader = _prependHeaderToData(blockStore->knownBlockVersions()->myClientId(), version, std::move(data));
auto baseBlock = baseBlockStore->overwrite(key, dataWithHeader.copy()); // TODO Copy necessary?
return cpputils::make_unique_ref<VersionCountingBlock>(std::move(baseBlock), std::move(dataWithHeader), blockStore);
}
inline cpputils::Data VersionCountingBlock::_prependHeaderToData(uint32_t myClientId, uint64_t version, cpputils::Data data) {
static_assert(HEADER_LENGTH == sizeof(FORMAT_VERSION_HEADER) + sizeof(myClientId) + sizeof(version), "Wrong header length");
cpputils::Data result(data.size() + HEADER_LENGTH);
std::memcpy(result.dataOffset(0), &FORMAT_VERSION_HEADER, sizeof(FORMAT_VERSION_HEADER));
std::memcpy(result.dataOffset(CLIENTID_HEADER_OFFSET), &myClientId, sizeof(myClientId));
std::memcpy(result.dataOffset(VERSION_HEADER_OFFSET), &version, sizeof(version));
std::memcpy((uint8_t*)result.dataOffset(HEADER_LENGTH), data.data(), data.size());
return result;
}
inline cpputils::unique_ref<VersionCountingBlock> VersionCountingBlock::Load(cpputils::unique_ref<Block> baseBlock, VersionCountingBlockStore *blockStore) {
cpputils::Data data(baseBlock->size());
std::memcpy(data.data(), baseBlock->data(), data.size());
_checkFormatHeader(data);
auto block = cpputils::make_unique_ref<VersionCountingBlock>(std::move(baseBlock), std::move(data), blockStore);
block->_checkVersion();
return std::move(block);
}
inline void VersionCountingBlock::_checkVersion() {
uint32_t lastClientId = _readClientId();
uint64_t version = _readVersion();
if(!_blockStore->knownBlockVersions()->checkAndUpdateVersion(lastClientId, key(), version)) {
_blockStore->integrityViolationDetected("The block version number is too low. Did an attacker try to roll back the block or to re-introduce a deleted block?");
}
}
inline void VersionCountingBlock::_checkFormatHeader(const cpputils::Data &data) {
if (*reinterpret_cast<decltype(FORMAT_VERSION_HEADER)*>(data.data()) != FORMAT_VERSION_HEADER) {
throw std::runtime_error("The versioned block has the wrong format. Was it created with a newer version of CryFS?");
}
}
inline uint32_t VersionCountingBlock::_readClientId() {
uint32_t clientId;
std::memcpy(&clientId, _dataWithHeader.dataOffset(CLIENTID_HEADER_OFFSET), sizeof(clientId));
return clientId;
}
inline uint64_t VersionCountingBlock::_readVersion() {
uint64_t version;
std::memcpy(&version, _dataWithHeader.dataOffset(VERSION_HEADER_OFFSET), sizeof(version));
return version;
}
inline VersionCountingBlock::VersionCountingBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data dataWithHeader, VersionCountingBlockStore *blockStore)
:Block(baseBlock->key()),
_blockStore(blockStore),
_baseBlock(std::move(baseBlock)),
_dataWithHeader(std::move(dataWithHeader)),
_dataChanged(false),
_mutex() {
if (_readVersion() == std::numeric_limits<uint64_t>::max()) {
throw std::runtime_error("Version overflow when loading. This shouldn't happen because in case of a version number overflow, the block isn't stored at all.");
}
}
inline VersionCountingBlock::~VersionCountingBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_storeToBaseBlock();
}
inline const void *VersionCountingBlock::data() const {
return (uint8_t*)_dataWithHeader.data() + HEADER_LENGTH;
}
inline void VersionCountingBlock::write(const void *source, uint64_t offset, uint64_t count) {
ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition
std::memcpy((uint8_t*)_dataWithHeader.data()+HEADER_LENGTH+offset, source, count);
_dataChanged = true;
}
inline void VersionCountingBlock::flush() {
std::unique_lock<std::mutex> lock(_mutex);
_storeToBaseBlock();
return _baseBlock->flush();
}
inline size_t VersionCountingBlock::size() const {
return _dataWithHeader.size() - HEADER_LENGTH;
}
inline void VersionCountingBlock::resize(size_t newSize) {
_dataWithHeader = cpputils::DataUtils::resize(std::move(_dataWithHeader), newSize + HEADER_LENGTH);
_dataChanged = true;
}
inline void VersionCountingBlock::_storeToBaseBlock() {
if (_dataChanged) {
uint64_t version = _blockStore->knownBlockVersions()->incrementVersion(key());
uint32_t myClientId = _blockStore->knownBlockVersions()->myClientId();
std::memcpy(_dataWithHeader.dataOffset(CLIENTID_HEADER_OFFSET), &myClientId, sizeof(myClientId));
std::memcpy(_dataWithHeader.dataOffset(VERSION_HEADER_OFFSET), &version, sizeof(version));
if (_baseBlock->size() != _dataWithHeader.size()) {
_baseBlock->resize(_dataWithHeader.size());
}
_baseBlock->write(_dataWithHeader.data(), 0, _dataWithHeader.size());
_dataChanged = false;
}
}
inline cpputils::unique_ref<Block> VersionCountingBlock::releaseBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_storeToBaseBlock();
return std::move(_baseBlock);
}
inline uint64_t VersionCountingBlock::blockSizeFromPhysicalBlockSize(uint64_t blockSize) {
if (blockSize <= HEADER_LENGTH) {
return 0;
}
return blockSize - HEADER_LENGTH;
}
}
}
#endif

103
src/blockstore/implementations/versioncounting/VersionCountingBlockStore.cpp
View File

@ -1,103 +0,0 @@
#include <unordered_set>
#include "VersionCountingBlockStore.h"
#include "VersionCountingBlock.h"
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using cpputils::Data;
using boost::none;
using boost::optional;
using std::string;
namespace bf = boost::filesystem;
namespace blockstore {
namespace versioncounting {
VersionCountingBlockStore::VersionCountingBlockStore(unique_ref<BlockStore> baseBlockStore, const bf::path &integrityFilePath, uint32_t myClientId, bool missingBlockIsIntegrityViolation)
: _baseBlockStore(std::move(baseBlockStore)), _knownBlockVersions(integrityFilePath, myClientId), _missingBlockIsIntegrityViolation(missingBlockIsIntegrityViolation), _integrityViolationDetected(false) {
}
Key VersionCountingBlockStore::createKey() {
return _baseBlockStore->createKey();
}
optional<unique_ref<Block>> VersionCountingBlockStore::tryCreate(const Key &key, Data data) {
_checkNoPastIntegrityViolations();
//TODO Easier implementation? This is only so complicated because of the cast VersionCountingBlock -> Block
auto result = VersionCountingBlock::TryCreateNew(_baseBlockStore.get(), key, std::move(data), this);
if (result == boost::none) {
return boost::none;
}
return unique_ref<Block>(std::move(*result));
}
unique_ref<Block> VersionCountingBlockStore::overwrite(const Key &key, Data data) {
_checkNoPastIntegrityViolations();
return VersionCountingBlock::Overwrite(_baseBlockStore.get(), key, std::move(data), this);
}
optional<unique_ref<Block>> VersionCountingBlockStore::load(const Key &key) {
_checkNoPastIntegrityViolations();
auto block = _baseBlockStore->load(key);
if (block == boost::none) {
if (_missingBlockIsIntegrityViolation && _knownBlockVersions.blockShouldExist(key)) {
integrityViolationDetected("A block that should exist wasn't found. Did an attacker delete it?");
}
return boost::none;
}
return optional<unique_ref<Block>>(VersionCountingBlock::Load(std::move(*block), this));
}
void VersionCountingBlockStore::_checkNoPastIntegrityViolations() {
if (_integrityViolationDetected) {
throw std::runtime_error(string() +
"There was an integrity violation detected. Preventing any further access to the file system. " +
"If you want to reset the integrity data (i.e. accept changes made by a potential attacker), " +
"please unmount the file system and delete the following file before re-mounting it: " +
_knownBlockVersions.path().native());
}
}
void VersionCountingBlockStore::integrityViolationDetected(const string &reason) const {
_integrityViolationDetected = true;
throw IntegrityViolationError(reason);
}
void VersionCountingBlockStore::remove(const Key &key) {
_knownBlockVersions.markBlockAsDeleted(key);
_baseBlockStore->remove(key);
}
uint64_t VersionCountingBlockStore::numBlocks() const {
return _baseBlockStore->numBlocks();
}
uint64_t VersionCountingBlockStore::estimateNumFreeBytes() const {
return _baseBlockStore->estimateNumFreeBytes();
}
uint64_t VersionCountingBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return VersionCountingBlock::blockSizeFromPhysicalBlockSize(_baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize));
}
void VersionCountingBlockStore::forEachBlock(std::function<void (const Key &)> callback) const {
if (!_missingBlockIsIntegrityViolation) {
return _baseBlockStore->forEachBlock(callback);
}
std::unordered_set<blockstore::Key> existingBlocks = _knownBlockVersions.existingBlocks();
_baseBlockStore->forEachBlock([&existingBlocks, callback] (const Key &key) {
callback(key);
auto found = existingBlocks.find(key);
if (found != existingBlocks.end()) {
existingBlocks.erase(found);
}
});
if (!existingBlocks.empty()) {
integrityViolationDetected("A block that should have existed wasn't found.");
}
}
}
}

49
src/blockstore/implementations/versioncounting/VersionCountingBlockStore.h
View File

@ -1,49 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_VERSIONCOUNTING_VERSIONCOUNTINGBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_VERSIONCOUNTING_VERSIONCOUNTINGBLOCKSTORE_H_
#include "../../interface/BlockStore.h"
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/cast.h>
#include "KnownBlockVersions.h"
#include <iostream>
namespace blockstore {
namespace versioncounting {
class VersionCountingBlockStore final: public BlockStore {
public:
VersionCountingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId, bool missingBlockIsIntegrityViolation);
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
cpputils::unique_ref<Block> overwrite(const blockstore::Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(const Key &key) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const Key &)> callback) const override;
void integrityViolationDetected(const std::string &reason) const;
KnownBlockVersions *knownBlockVersions();
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
KnownBlockVersions _knownBlockVersions;
const bool _missingBlockIsIntegrityViolation;
mutable bool _integrityViolationDetected;
void _checkNoPastIntegrityViolations();
DISALLOW_COPY_AND_ASSIGN(VersionCountingBlockStore);
};
inline KnownBlockVersions *VersionCountingBlockStore::knownBlockVersions() {
return &_knownBlockVersions;
}
}
}
#endif

17
src/blockstore/implementations/versioncounting/VersionCountingBlockStore2.cpp
View File

@ -7,7 +7,6 @@ using cpputils::unique_ref;
using std::string;
using boost::optional;
using boost::none;
using boost::future;
using namespace cpputils::logging;
namespace blockstore {
@ -78,23 +77,22 @@ VersionCountingBlockStore2::VersionCountingBlockStore2(unique_ref<BlockStore2> b
: _baseBlockStore(std::move(baseBlockStore)), _knownBlockVersions(integrityFilePath, myClientId), _missingBlockIsIntegrityViolation(missingBlockIsIntegrityViolation), _integrityViolationDetected(false) {
}
future<bool> VersionCountingBlockStore2::tryCreate(const Key &key, const Data &data) {
bool VersionCountingBlockStore2::tryCreate(const Key &key, const Data &data) {
_checkNoPastIntegrityViolations();
uint64_t version = _knownBlockVersions.incrementVersion(key);
Data dataWithHeader = _prependHeaderToData(_knownBlockVersions.myClientId(), version, data);
return _baseBlockStore->tryCreate(key, dataWithHeader);
}
future<bool> VersionCountingBlockStore2::remove(const Key &key) {
bool VersionCountingBlockStore2::remove(const Key &key) {
_checkNoPastIntegrityViolations();
_knownBlockVersions.markBlockAsDeleted(key);
return _baseBlockStore->remove(key);
}
future<optional<Data>> VersionCountingBlockStore2::load(const Key &key) const {
optional<Data> VersionCountingBlockStore2::load(const Key &key) const {
_checkNoPastIntegrityViolations();
return _baseBlockStore->load(key).then([this, key] (future<optional<Data>> loaded_) {
auto loaded = loaded_.get();
auto loaded = _baseBlockStore->load(key);
if (none == loaded) {
if (_missingBlockIsIntegrityViolation && _knownBlockVersions.blockShouldExist(key)) {
integrityViolationDetected("A block that should exist wasn't found. Did an attacker delete it?");
@ -103,10 +101,9 @@ future<optional<Data>> VersionCountingBlockStore2::load(const Key &key) const {
}
_checkHeader(key, *loaded);
return optional<Data>(_removeHeader(*loaded));
});
}
future<void> VersionCountingBlockStore2::store(const Key &key, const Data &data) {
void VersionCountingBlockStore2::store(const Key &key, const Data &data) {
_checkNoPastIntegrityViolations();
uint64_t version = _knownBlockVersions.incrementVersion(key);
Data dataWithHeader = _prependHeaderToData(_knownBlockVersions.myClientId(), version, data);
@ -161,14 +158,14 @@ void VersionCountingBlockStore2::migrateFromBlockstoreWithoutVersionNumbers(Bloc
void VersionCountingBlockStore2::migrateBlockFromBlockstoreWithoutVersionNumbers(blockstore::BlockStore2* baseBlockStore, const blockstore::Key& key, KnownBlockVersions *knownBlockVersions) {
uint64_t version = knownBlockVersions->incrementVersion(key);
auto data_ = baseBlockStore->load(key).get(); // TODO this is blocking
auto data_ = baseBlockStore->load(key);
if (data_ == boost::none) {
LOG(WARN, "Block not found, but was returned from forEachBlock before");
return;
}
cpputils::Data data = std::move(*data_);
cpputils::Data dataWithHeader = _prependHeaderToData(knownBlockVersions->myClientId(), version, std::move(data));
baseBlockStore->store(key, std::move(dataWithHeader)).wait(); // TODO This is blocking
baseBlockStore->store(key, std::move(dataWithHeader));
}
#endif

8
src/blockstore/implementations/versioncounting/VersionCountingBlockStore2.h
View File

@ -16,10 +16,10 @@ class VersionCountingBlockStore2 final: public BlockStore2 {
public:
VersionCountingBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId, bool missingBlockIsIntegrityViolation);
boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) override;
boost::future<bool> remove(const Key &key) override;
boost::future<boost::optional<cpputils::Data>> load(const Key &key) const override;
boost::future<void> store(const Key &key, const cpputils::Data &data) override;
bool tryCreate(const Key &key, const cpputils::Data &data) override;
bool remove(const Key &key) override;
boost::optional<cpputils::Data> load(const Key &key) const override;
void store(const Key &key, const cpputils::Data &data) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;

1
src/blockstore/interface/BlockStore.h
View File

@ -16,6 +16,7 @@ public:
virtual Key createKey() = 0;
//Returns boost::none if key already exists
// TODO Can we make data passed in by ref?
virtual boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) = 0;
//TODO Use boost::optional (if key doesn't exist)
// Return nullptr if block with this key doesn't exists

25
src/blockstore/interface/BlockStore2.h
View File

@ -7,11 +7,8 @@
#include <boost/optional.hpp>
#include <cpp-utils/pointer/unique_ref.h>
#include <cpp-utils/data/Data.h>
#include <boost/thread/future.hpp>
#include <cpp-utils/random/Random.h>
// TODO warn_unused_result for all boost::future interfaces
namespace blockstore {
class BlockStore2 {
@ -19,28 +16,24 @@ public:
virtual ~BlockStore2() {}
__attribute__((warn_unused_result))
virtual boost::future<bool> tryCreate(const Key &key, const cpputils::Data &data) = 0;
virtual bool tryCreate(const Key &key, const cpputils::Data &data) = 0;
__attribute__((warn_unused_result))
virtual boost::future<bool> remove(const Key &key) = 0;
virtual bool remove(const Key &key) = 0;
__attribute__((warn_unused_result))
virtual boost::future<boost::optional<cpputils::Data>> load(const Key &key) const = 0;
virtual boost::optional<cpputils::Data> load(const Key &key) const = 0;
// Store the block with the given key. If it doesn't exist, it is created.
__attribute__((warn_unused_result))
virtual boost::future<void> store(const Key &key, const cpputils::Data &data) = 0;
virtual void store(const Key &key, const cpputils::Data &data) = 0;
__attribute__((warn_unused_result))
boost::future<Key> create(cpputils::Data data) {
Key create(const cpputils::Data& data) {
Key key = cpputils::Random::PseudoRandom().getFixedSize<Key::BINARY_LENGTH>();
boost::future<bool> successFuture = tryCreate(key, data);
return successFuture.then([this, key, data = std::move(data)] (boost::future<bool> success) mutable {
if (success.get()) {
return boost::make_ready_future<Key>(key);
bool success = tryCreate(key, data);
if (success) {
return key;
} else {
return this->create(std::move(data));
return create(data);
}
});
}
virtual uint64_t numBlocks() const = 0;

3
src/cryfs-cli/Cli.cpp
View File

@ -1,8 +1,6 @@
#include "Cli.h"
#include <blockstore/implementations/ondisk/OnDiskBlockStore2.h>
#include <blockstore/implementations/inmemory/InMemoryBlockStore.h>
#include <blockstore/implementations/inmemory/InMemoryBlock.h>
#include <cmath>
#include <cstdio>
#include <cstdlib>
@ -33,7 +31,6 @@ namespace bf = boost::filesystem;
using namespace cpputils::logging;
using blockstore::ondisk::OnDiskBlockStore2;
using blockstore::inmemory::InMemoryBlockStore;
using program_options::ProgramOptions;
using cpputils::make_unique_ref;

1
src/cryfs/filesystem/CryDevice.cpp
View File

@ -12,7 +12,6 @@
#include <blobstore/implementations/onblocks/BlobOnBlocks.h>
#include <blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h>
#include <blockstore/implementations/encrypted/EncryptedBlockStore2.h>
#include <blockstore/implementations/versioncounting/VersionCountingBlock.h>
#include <blockstore/implementations/versioncounting/VersionCountingBlockStore2.h>
#include "parallelaccessfsblobstore/ParallelAccessFsBlobStore.h"
#include "cachingfsblobstore/CachingFsBlobStore.h"

9
test/blobstore/implementations/onblocks/BigBlobsTest.cpp
View File

@ -1,8 +1,8 @@
#include <gtest/gtest.h>
#include <blockstore/implementations/compressing/CompressingBlockStore.h>
#include <blockstore/implementations/compressing/compressors/RunLengthEncoding.h>
#include <blockstore/implementations/inmemory/InMemoryBlockStore.h>
#include <blockstore/implementations/inmemory/InMemoryBlock.h>
#include <blockstore/implementations/inmemory/InMemoryBlockStore2.h>
#include <blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h>
#include <cpp-utils/data/DataFixture.h>
#include <cpp-utils/data/Data.h>
#include "blobstore/implementations/onblocks/BlobStoreOnBlocks.h"
@ -14,7 +14,8 @@ using cpputils::unique_ref;
using cpputils::make_unique_ref;
using cpputils::DataFixture;
using cpputils::Data;
using blockstore::inmemory::InMemoryBlockStore;
using blockstore::inmemory::InMemoryBlockStore2;
using blockstore::lowtohighlevel::LowToHighLevelBlockStore;
using blockstore::compressing::CompressingBlockStore;
using blockstore::compressing::RunLengthEncoding;
@ -29,7 +30,7 @@ public:
static_assert(SMALL_BLOB_SIZE < max_uint_32, "LARGE_BLOB_SIZE should need 64bit or the test case is mute");
static_assert(LARGE_BLOB_SIZE > max_uint_32, "LARGE_BLOB_SIZE should need 64bit or the test case is mute");
unique_ref<BlobStore> blobStore = make_unique_ref<BlobStoreOnBlocks>(make_unique_ref<CompressingBlockStore<RunLengthEncoding>>(make_unique_ref<InMemoryBlockStore>()), BLOCKSIZE);
unique_ref<BlobStore> blobStore = make_unique_ref<BlobStoreOnBlocks>(make_unique_ref<CompressingBlockStore<RunLengthEncoding>>(make_unique_ref<LowToHighLevelBlockStore>(make_unique_ref<InMemoryBlockStore2>())), BLOCKSIZE);
unique_ref<Blob> blob = blobStore->create();
};

9
test/blockstore/implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp
View File

@ -1,6 +1,6 @@
#include <cpp-utils/crypto/symmetric/ciphers.h>
#include <cpp-utils/crypto/symmetric/Cipher.h>
#include "blockstore/implementations/encrypted/EncryptedBlockStore.h"
#include <blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h>
#include "blockstore/implementations/encrypted/EncryptedBlockStore2.h"
#include "blockstore/implementations/testfake/FakeBlockStore.h"
#include "blockstore/implementations/inmemory/InMemoryBlockStore2.h"
@ -14,9 +14,8 @@ using ::testing::Test;
using blockstore::BlockStore;
using blockstore::BlockStore2;
using blockstore::encrypted::EncryptedBlockStore;
using blockstore::encrypted::EncryptedBlockStore2;
using blockstore::testfake::FakeBlockStore;
using blockstore::lowtohighlevel::LowToHighLevelBlockStore;
using blockstore::inmemory::InMemoryBlockStore2;
using cpputils::AES256_GCM;
using cpputils::AES256_CFB;
@ -31,7 +30,9 @@ template<class Cipher>
class EncryptedBlockStoreTestFixture: public BlockStoreTestFixture {
public:
unique_ref<BlockStore> createBlockStore() override {
return make_unique_ref<EncryptedBlockStore<Cipher>>(make_unique_ref<FakeBlockStore>(), createKeyFixture());
return make_unique_ref<LowToHighLevelBlockStore>(
make_unique_ref<EncryptedBlockStore2<Cipher>>(make_unique_ref<InMemoryBlockStore2>(), createKeyFixture())
);
}
private:

44
test/blockstore/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp
View File

@ -1,7 +1,7 @@
#include <gtest/gtest.h>
#include "../../../cpp-utils/crypto/symmetric/testutils/FakeAuthenticatedCipher.h"
#include "blockstore/implementations/encrypted/EncryptedBlockStore.h"
#include "blockstore/implementations/testfake/FakeBlockStore.h"
#include "blockstore/implementations/encrypted/EncryptedBlockStore2.h"
#include "blockstore/implementations/inmemory/InMemoryBlockStore2.h"
#include "blockstore/utils/BlockStoreUtils.h"
#include <cpp-utils/data/DataFixture.h>
@ -13,7 +13,7 @@ using cpputils::unique_ref;
using cpputils::make_unique_ref;
using cpputils::FakeAuthenticatedCipher;
using blockstore::testfake::FakeBlockStore;
using blockstore::inmemory::InMemoryBlockStore2;
using namespace blockstore::encrypted;
@ -21,12 +21,12 @@ class EncryptedBlockStoreTest: public Test {
public:
static constexpr unsigned int BLOCKSIZE = 1024;
EncryptedBlockStoreTest():
baseBlockStore(new FakeBlockStore),
blockStore(make_unique_ref<EncryptedBlockStore<FakeAuthenticatedCipher>>(std::move(cpputils::nullcheck(std::unique_ptr<FakeBlockStore>(baseBlockStore)).value()), FakeAuthenticatedCipher::Key1())),
baseBlockStore(new InMemoryBlockStore2),
blockStore(make_unique_ref<EncryptedBlockStore2<FakeAuthenticatedCipher>>(std::move(cpputils::nullcheck(std::unique_ptr<InMemoryBlockStore2>(baseBlockStore)).value()), FakeAuthenticatedCipher::Key1())),
data(DataFixture::generate(BLOCKSIZE)) {
}
FakeBlockStore *baseBlockStore;
unique_ref<EncryptedBlockStore<FakeAuthenticatedCipher>> blockStore;
InMemoryBlockStore2 *baseBlockStore;
unique_ref<EncryptedBlockStore2<FakeAuthenticatedCipher>> blockStore;
Data data;
blockstore::Key CreateBlockDirectlyWithFixtureAndReturnKey() {
@ -34,25 +34,25 @@ public:
}
blockstore::Key CreateBlockReturnKey(const Data &initData) {
return blockStore->create(initData)->key();
return blockStore->create(initData.copy());
}
blockstore::Key CreateBlockWriteFixtureToItAndReturnKey() {
auto block = blockStore->create(Data(data.size()));
block->write(data.data(), 0, data.size());
return block->key();
auto key = blockStore->create(Data(data.size()));
blockStore->store(key, data);
return key;
}
void ModifyBaseBlock(const blockstore::Key &key) {
auto block = baseBlockStore->load(key).value();
uint8_t middle_byte = ((byte*)block->data())[10];
uint8_t new_middle_byte = middle_byte + 1;
block->write(&new_middle_byte, 10, 1);
byte* middle_byte = ((byte*)block.data()) + 10;
*middle_byte = *middle_byte + 1;
baseBlockStore->store(key, block);
}
blockstore::Key CopyBaseBlock(const blockstore::Key &key) {
auto source = baseBlockStore->load(key).value();
return blockstore::utils::copyToNewBlock(baseBlockStore, *source)->key();
return baseBlockStore->create(std::move(source));
}
private:
@ -63,16 +63,16 @@ TEST_F(EncryptedBlockStoreTest, LoadingWithSameKeyWorks_WriteOnCreate) {
auto key = CreateBlockDirectlyWithFixtureAndReturnKey();
auto loaded = blockStore->load(key);
EXPECT_NE(boost::none, loaded);
EXPECT_EQ(data.size(), (*loaded)->size());
EXPECT_EQ(0, std::memcmp(data.data(), (*loaded)->data(), data.size()));
EXPECT_EQ(data.size(), loaded->size());
EXPECT_EQ(0, std::memcmp(data.data(), loaded->data(), data.size()));
}
TEST_F(EncryptedBlockStoreTest, LoadingWithSameKeyWorks_WriteSeparately) {
auto key = CreateBlockWriteFixtureToItAndReturnKey();
auto loaded = blockStore->load(key);
EXPECT_NE(boost::none, loaded);
EXPECT_EQ(data.size(), (*loaded)->size());
EXPECT_EQ(0, std::memcmp(data.data(), (*loaded)->data(), data.size()));
EXPECT_EQ(data.size(), loaded->size());
EXPECT_EQ(0, std::memcmp(data.data(), loaded->data(), data.size()));
}
TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentKeyDoesntWork_WriteOnCreate) {
@ -124,13 +124,13 @@ TEST_F(EncryptedBlockStoreTest, PhysicalBlockSize_zerophysical) {
TEST_F(EncryptedBlockStoreTest, PhysicalBlockSize_zerovirtual) {
auto key = CreateBlockReturnKey(Data(0));
auto base = baseBlockStore->load(key).value();
EXPECT_EQ(0u, blockStore->blockSizeFromPhysicalBlockSize(base->size()));
EXPECT_EQ(0u, blockStore->blockSizeFromPhysicalBlockSize(base.size()));
}
TEST_F(EncryptedBlockStoreTest, PhysicalBlockSize_negativeboundaries) {
// This tests that a potential if/else in blockSizeFromPhysicalBlockSize that catches negative values has the
// correct boundary set. We test the highest value that is negative and the smallest value that is positive.
auto physicalSizeForVirtualSizeZero = baseBlockStore->load(CreateBlockReturnKey(Data(0))).value()->size();
auto physicalSizeForVirtualSizeZero = baseBlockStore->load(CreateBlockReturnKey(Data(0))).value().size();
if (physicalSizeForVirtualSizeZero > 0) {
EXPECT_EQ(0u, blockStore->blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero - 1));
}
@ -141,5 +141,5 @@ TEST_F(EncryptedBlockStoreTest, PhysicalBlockSize_negativeboundaries) {
TEST_F(EncryptedBlockStoreTest, PhysicalBlockSize_positive) {
auto key = CreateBlockReturnKey(Data(10*1024));
auto base = baseBlockStore->load(key).value();
EXPECT_EQ(10*1024u, blockStore->blockSizeFromPhysicalBlockSize(base->size()));
EXPECT_EQ(10*1024u, blockStore->blockSizeFromPhysicalBlockSize(base.size()));
}

17
test/blockstore/implementations/inmemory/InMemoryBlockStoreTest.cpp
View File

@ -1,17 +1,16 @@
#include "blockstore/implementations/inmemory/InMemoryBlock.h"
#include "blockstore/implementations/inmemory/InMemoryBlockStore.h"
#include "blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h"
#include "blockstore/implementations/inmemory/InMemoryBlockStore2.h"
#include "../../testutils/BlockStoreTest.h"
#include "../../testutils/BlockStore2Test.h"
#include "../../testutils/BlockStoreWithRandomKeysTest.h"
#include <gtest/gtest.h>
#include <cpp-utils/pointer/unique_ref_boost_optional_gtest_workaround.h>
#include <blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h>
using blockstore::BlockStore;
using blockstore::BlockStore2;
using blockstore::BlockStoreWithRandomKeys;
using blockstore::inmemory::InMemoryBlockStore;
using blockstore::lowtohighlevel::LowToHighLevelBlockStore;
using blockstore::inmemory::InMemoryBlockStore2;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
@ -19,7 +18,9 @@ using cpputils::make_unique_ref;
class InMemoryBlockStoreTestFixture: public BlockStoreTestFixture {
public:
unique_ref<BlockStore> createBlockStore() override {
return make_unique_ref<InMemoryBlockStore>();
return make_unique_ref<LowToHighLevelBlockStore>(
make_unique_ref<InMemoryBlockStore2>()
);
}
};
@ -27,8 +28,10 @@ INSTANTIATE_TYPED_TEST_CASE_P(InMemory, BlockStoreTest, InMemoryBlockStoreTestFi
class InMemoryBlockStoreWithRandomKeysTestFixture: public BlockStoreWithRandomKeysTestFixture {
public:
unique_ref<BlockStoreWithRandomKeys> createBlockStore() override {
return make_unique_ref<InMemoryBlockStore>();
unique_ref<BlockStore> createBlockStore() override {
return make_unique_ref<LowToHighLevelBlockStore>(
make_unique_ref<InMemoryBlockStore2>()
);
}
};

19
test/blockstore/implementations/ondisk/OnDiskBlockStoreTest_Generic.cpp
View File

@ -1,5 +1,4 @@
#include "blockstore/implementations/ondisk/OnDiskBlock.h"
#include "blockstore/implementations/ondisk/OnDiskBlockStore.h"
#include "blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h"
#include "blockstore/implementations/ondisk/OnDiskBlockStore2.h"
#include "../../testutils/BlockStoreTest.h"
#include "../../testutils/BlockStore2Test.h"
@ -10,10 +9,9 @@
using blockstore::BlockStore;
using blockstore::BlockStoreWithRandomKeys;
using blockstore::ondisk::OnDiskBlockStore;
using blockstore::BlockStore2;
using blockstore::ondisk::OnDiskBlockStore2;
using blockstore::BlockStore2;
using blockstore::lowtohighlevel::LowToHighLevelBlockStore;
using cpputils::TempDir;
using cpputils::unique_ref;
@ -24,7 +22,9 @@ public:
OnDiskBlockStoreTestFixture(): tempdir() {}
unique_ref<BlockStore> createBlockStore() override {
return make_unique_ref<OnDiskBlockStore>(tempdir.path());
return make_unique_ref<LowToHighLevelBlockStore>(
make_unique_ref<OnDiskBlockStore2>(tempdir.path())
);
}
private:
TempDir tempdir;
@ -32,12 +32,15 @@ private:
INSTANTIATE_TYPED_TEST_CASE_P(OnDisk, BlockStoreTest, OnDiskBlockStoreTestFixture);
// TODO Add BlockStoreWithRandomKeysTest to BlockStoreTest and BlockStore2Test
class OnDiskBlockStoreWithRandomKeysTestFixture: public BlockStoreWithRandomKeysTestFixture {
public:
OnDiskBlockStoreWithRandomKeysTestFixture(): tempdir() {}
unique_ref<BlockStoreWithRandomKeys> createBlockStore() override {
return make_unique_ref<OnDiskBlockStore>(tempdir.path());
unique_ref<BlockStore> createBlockStore() override {
return make_unique_ref<LowToHighLevelBlockStore>(
make_unique_ref<OnDiskBlockStore2>(tempdir.path())
);
}
private:
TempDir tempdir;

10
test/blockstore/implementations/ondisk/OnDiskBlockStoreTest_Specific.cpp
View File

@ -1,5 +1,5 @@
#include <gtest/gtest.h>
#include "blockstore/implementations/ondisk/OnDiskBlockStore.h"
#include "blockstore/implementations/ondisk/OnDiskBlockStore2.h"
#include <cpp-utils/tempfile/TempDir.h>
using ::testing::Test;
@ -18,10 +18,10 @@ public:
blockStore(baseDir.path()) {
}
TempDir baseDir;
OnDiskBlockStore blockStore;
OnDiskBlockStore2 blockStore;
blockstore::Key CreateBlockReturnKey(const Data &initData) {
return blockStore.create(initData)->key();
return blockStore.create(initData.copy());
}
uint64_t getPhysicalBlockSize(const Key &key) {
@ -61,7 +61,7 @@ TEST_F(OnDiskBlockStoreTest, PhysicalBlockSize_positive) {
TEST_F(OnDiskBlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocksWithSameKeyPrefix) {
const Key key1 = Key::FromString("4CE72ECDD20877A12ADBF4E3927C0A13");
const Key key2 = Key::FromString("4CE72ECDD20877A12ADBF4E3927C0A14");
EXPECT_NE(boost::none, blockStore.tryCreate(key1, cpputils::Data(0)));
EXPECT_NE(boost::none, blockStore.tryCreate(key2, cpputils::Data(0)));
EXPECT_TRUE(blockStore.tryCreate(key1, cpputils::Data(0)));
EXPECT_TRUE(blockStore.tryCreate(key2, cpputils::Data(0)));
EXPECT_EQ(2u, blockStore.numBlocks());
}

5
test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp
View File

@ -1,9 +1,11 @@
#include "blockstore/implementations/ondisk/OnDiskBlock.h"
#include <gtest/gtest.h>
#include <cpp-utils/tempfile/TempFile.h>
#include <cpp-utils/tempfile/TempDir.h>
// TODO This should be ported to BlockStore2
/*
using ::testing::Test;
using ::testing::WithParamInterface;
using ::testing::Values;
@ -83,3 +85,4 @@ TEST_P(OnDiskBlockCreateSizeTest, InMemorySizeIsCorrect) {
TEST_P(OnDiskBlockCreateSizeTest, InMemoryBlockIsZeroedOut) {
EXPECT_EQ(0, std::memcmp(ZEROES.data(), block->data(), block->size()));
}
*/

4
test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp
View File

@ -1,10 +1,11 @@
#include "blockstore/implementations/ondisk/OnDiskBlock.h"
#include <cpp-utils/data/DataFixture.h>
#include <gtest/gtest.h>
#include <cpp-utils/tempfile/TempFile.h>
#include <cpp-utils/tempfile/TempDir.h>
// TODO This should be ported to BlockStore2
/*
using ::testing::Test;
using ::testing::WithParamInterface;
using ::testing::Values;
@ -115,3 +116,4 @@ TEST_P(OnDiskBlockFlushTest, AfterLoad_FlushesWhenDestructed) {
}
EXPECT_STORED_FILE_DATA_CORRECT();
}
*/

4
test/blockstore/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp
View File

@ -1,4 +1,3 @@
#include "blockstore/implementations/ondisk/OnDiskBlock.h"
#include <cpp-utils/data/DataFixture.h>
#include "blockstore/utils/FileDoesntExistException.h"
#include <gtest/gtest.h>
@ -9,6 +8,8 @@
#include <cpp-utils/pointer/unique_ref_boost_optional_gtest_workaround.h>
#include <fstream>
// TODO This should be ported to BlockStore2
/*
using ::testing::Test;
using ::testing::WithParamInterface;
using ::testing::Values;
@ -78,3 +79,4 @@ TEST_F(OnDiskBlockLoadTest, LoadNotExistingBlock) {
Key key2 = Key::FromString("272EE5517627CFA147A971A8E6E747E0");
EXPECT_EQ(boost::none, OnDiskBlock::LoadFromDisk(dir.path(), key2));
}
*/

2
test/blockstore/implementations/testfake/TestFakeBlockStoreTest.cpp
View File

@ -22,7 +22,7 @@ INSTANTIATE_TYPED_TEST_CASE_P(TestFake, BlockStoreTest, FakeBlockStoreTestFixtur
class FakeBlockStoreWithRandomKeysTestFixture: public BlockStoreWithRandomKeysTestFixture {
public:
unique_ref<BlockStoreWithRandomKeys> createBlockStore() override {
unique_ref<BlockStore> createBlockStore() override {
return make_unique_ref<FakeBlockStore>();
}
};

2
test/blockstore/implementations/versioncounting/KnownBlockVersionsTest.cpp
View File

@ -1,10 +1,8 @@
#include <gtest/gtest.h>
#include <blockstore/implementations/versioncounting/KnownBlockVersions.h>
#include <cpp-utils/tempfile/TempFile.h>
#include <blockstore/implementations/versioncounting/VersionCountingBlock.h>
using blockstore::versioncounting::KnownBlockVersions;
using blockstore::versioncounting::VersionCountingBlock;
using blockstore::Key;
using cpputils::TempFile;
using std::unordered_set;

12
test/blockstore/implementations/versioncounting/VersionCountingBlockStoreTest_Generic.cpp
View File

@ -1,6 +1,5 @@
#include "blockstore/implementations/versioncounting/VersionCountingBlockStore.h"
#include "blockstore/implementations/versioncounting/VersionCountingBlockStore2.h"
#include "blockstore/implementations/testfake/FakeBlockStore.h"
#include "blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h"
#include "blockstore/implementations/inmemory/InMemoryBlockStore2.h"
#include "../../testutils/BlockStoreTest.h"
#include "../../testutils/BlockStore2Test.h"
@ -11,10 +10,9 @@ using ::testing::Test;
using blockstore::BlockStore;
using blockstore::BlockStore2;
using blockstore::versioncounting::VersionCountingBlockStore;
using blockstore::versioncounting::VersionCountingBlockStore2;
using blockstore::versioncounting::KnownBlockVersions;
using blockstore::testfake::FakeBlockStore;
using blockstore::lowtohighlevel::LowToHighLevelBlockStore;
using blockstore::inmemory::InMemoryBlockStore2;
using cpputils::Data;
@ -30,10 +28,14 @@ public:
TempFile stateFile;
unique_ref<BlockStore> createBlockStore() override {
return make_unique_ref<VersionCountingBlockStore>(make_unique_ref<FakeBlockStore>(), stateFile.path(), 0x12345678, MissingBlockIsIntegrityViolation);
return make_unique_ref<LowToHighLevelBlockStore>(
make_unique_ref<VersionCountingBlockStore2>(make_unique_ref<InMemoryBlockStore2>(), stateFile.path(), 0x12345678, MissingBlockIsIntegrityViolation)
);
}
};
// TODO Why is here no VersionCountingBlockStoreWithRandomKeysTest?
INSTANTIATE_TYPED_TEST_CASE_P(VersionCounting_multiclient, BlockStoreTest, VersionCountingBlockStoreTestFixture<false>);
INSTANTIATE_TYPED_TEST_CASE_P(VersionCounting_singleclient, BlockStoreTest, VersionCountingBlockStoreTestFixture<true>);

100
test/blockstore/implementations/versioncounting/VersionCountingBlockStoreTest_Specific.cpp
View File

@ -1,7 +1,6 @@
#include <gtest/gtest.h>
#include "blockstore/implementations/versioncounting/VersionCountingBlockStore.h"
#include "blockstore/implementations/versioncounting/VersionCountingBlock.h"
#include "blockstore/implementations/testfake/FakeBlockStore.h"
#include "blockstore/implementations/versioncounting/VersionCountingBlockStore2.h"
#include "blockstore/implementations/inmemory/InMemoryBlockStore2.h"
#include "blockstore/utils/BlockStoreUtils.h"
#include <cpp-utils/data/DataFixture.h>
#include <cpp-utils/tempfile/TempFile.h>
@ -17,7 +16,7 @@ using boost::none;
using std::make_unique;
using std::unique_ptr;
using blockstore::testfake::FakeBlockStore;
using blockstore::inmemory::InMemoryBlockStore2;
using namespace blockstore::versioncounting;
@ -26,25 +25,25 @@ public:
static constexpr unsigned int BLOCKSIZE = 1024;
VersionCountingBlockStoreTest():
stateFile(false),
baseBlockStore(new FakeBlockStore),
blockStore(make_unique_ref<VersionCountingBlockStore>(std::move(cpputils::nullcheck(std::unique_ptr<FakeBlockStore>(baseBlockStore)).value()), stateFile.path(), myClientId, false)),
baseBlockStore(new InMemoryBlockStore2),
blockStore(make_unique_ref<VersionCountingBlockStore2>(std::move(cpputils::nullcheck(std::unique_ptr<InMemoryBlockStore2>(baseBlockStore)).value()), stateFile.path(), myClientId, false)),
data(DataFixture::generate(BLOCKSIZE)) {
}
static constexpr uint32_t myClientId = 0x12345678;
TempFile stateFile;
FakeBlockStore *baseBlockStore;
unique_ref<VersionCountingBlockStore> blockStore;
InMemoryBlockStore2 *baseBlockStore;
unique_ref<VersionCountingBlockStore2> blockStore;
Data data;
std::pair<FakeBlockStore *, unique_ptr<VersionCountingBlockStore>> makeBlockStoreWithDeletionPrevention() {
FakeBlockStore *baseBlockStore = new FakeBlockStore;
auto blockStore = make_unique<VersionCountingBlockStore>(std::move(cpputils::nullcheck(std::unique_ptr<FakeBlockStore>(baseBlockStore)).value()), stateFile.path(), myClientId, true);
std::pair<InMemoryBlockStore2 *, unique_ptr<VersionCountingBlockStore2>> makeBlockStoreWithDeletionPrevention() {
InMemoryBlockStore2 *baseBlockStore = new InMemoryBlockStore2;
auto blockStore = make_unique<VersionCountingBlockStore2>(std::move(cpputils::nullcheck(std::unique_ptr<InMemoryBlockStore2>(baseBlockStore)).value()), stateFile.path(), myClientId, true);
return std::make_pair(baseBlockStore, std::move(blockStore));
}
std::pair<FakeBlockStore *, unique_ptr<VersionCountingBlockStore>> makeBlockStoreWithoutDeletionPrevention() {
FakeBlockStore *baseBlockStore = new FakeBlockStore;
auto blockStore = make_unique<VersionCountingBlockStore>(std::move(cpputils::nullcheck(std::unique_ptr<FakeBlockStore>(baseBlockStore)).value()), stateFile.path(), myClientId, false);
std::pair<InMemoryBlockStore2 *, unique_ptr<VersionCountingBlockStore2>> makeBlockStoreWithoutDeletionPrevention() {
InMemoryBlockStore2 *baseBlockStore = new InMemoryBlockStore2;
auto blockStore = make_unique<VersionCountingBlockStore2>(std::move(cpputils::nullcheck(std::unique_ptr<InMemoryBlockStore2>(baseBlockStore)).value()), stateFile.path(), myClientId, false);
return std::make_pair(baseBlockStore, std::move(blockStore));
}
@ -53,55 +52,48 @@ public:
}
blockstore::Key CreateBlockReturnKey(const Data &initData) {
return blockStore->create(initData)->key();
return blockStore->create(initData.copy());
}
Data loadBaseBlock(const blockstore::Key &key) {
auto block = baseBlockStore->load(key).value();
Data result(block->size());
std::memcpy(result.data(), block->data(), data.size());
return result;
return baseBlockStore->load(key).value();
}
Data loadBlock(const blockstore::Key &key) {
auto block = blockStore->load(key).value();
Data result(block->size());
std::memcpy(result.data(), block->data(), data.size());
return result;
return blockStore->load(key).value();
}
void modifyBlock(const blockstore::Key &key) {
auto block = blockStore->load(key).value();
uint64_t data = 5;
block->write(&data, 0, sizeof(data));
byte* first_byte = (byte*)block.data();
*first_byte = *first_byte + 1;
blockStore->store(key, block);
}
void rollbackBaseBlock(const blockstore::Key &key, const Data &data) {
auto block = baseBlockStore->load(key).value();
block->resize(data.size());
block->write(data.data(), 0, data.size());
baseBlockStore->store(key, data);
}
void decreaseVersionNumber(const blockstore::Key &key) {
auto baseBlock = baseBlockStore->load(key).value();
uint64_t version = *(uint64_t*)((uint8_t*)baseBlock->data()+VersionCountingBlock::VERSION_HEADER_OFFSET);
ASSERT(version > 1, "Can't decrease the lowest allowed version number");
version -= 1;
baseBlock->write((char*)&version, VersionCountingBlock::VERSION_HEADER_OFFSET, sizeof(version));
uint64_t* version = (uint64_t*)((uint8_t*)baseBlock.data()+VersionCountingBlockStore2::VERSION_HEADER_OFFSET);
ASSERT(*version > 1, "Can't decrease the lowest allowed version number");
*version -= 1;
baseBlockStore->store(key, baseBlock);
}
void increaseVersionNumber(const blockstore::Key &key) {
auto baseBlock = baseBlockStore->load(key).value();
uint64_t version = *(uint64_t*)((uint8_t*)baseBlock->data()+VersionCountingBlock::VERSION_HEADER_OFFSET);
version += 1;
baseBlock->write((char*)&version, VersionCountingBlock::VERSION_HEADER_OFFSET, sizeof(version));
uint64_t* version = (uint64_t*)((uint8_t*)baseBlock.data()+VersionCountingBlockStore2::VERSION_HEADER_OFFSET);
*version += 1;
baseBlockStore->store(key, baseBlock);
}
void changeClientId(const blockstore::Key &key) {
auto baseBlock = baseBlockStore->load(key).value();
uint32_t clientId = *(uint32_t*)((uint8_t*)baseBlock->data()+VersionCountingBlock::CLIENTID_HEADER_OFFSET);
clientId += 1;
baseBlock->write((char*)&clientId, VersionCountingBlock::CLIENTID_HEADER_OFFSET, sizeof(clientId));
uint32_t* clientId = (uint32_t*)((uint8_t*)baseBlock.data()+VersionCountingBlockStore2::CLIENTID_HEADER_OFFSET);
*clientId += 1;
baseBlockStore->store(key, baseBlock);
}
void deleteBlock(const blockstore::Key &key) {
@ -109,7 +101,7 @@ public:
}
void insertBaseBlock(const blockstore::Key &key, Data data) {
EXPECT_NE(none, baseBlockStore->tryCreate(key, std::move(data)));
EXPECT_TRUE(baseBlockStore->tryCreate(key, std::move(data)));
}
private:
@ -194,20 +186,20 @@ TEST_F(VersionCountingBlockStoreTest, RollbackPrevention_AllowsReintroducingDele
// Check that in a multi-client scenario, missing blocks are not integrity errors, because another client might have deleted them.
TEST_F(VersionCountingBlockStoreTest, DeletionPrevention_AllowsDeletingBlocksWhenDeactivated) {
FakeBlockStore *baseBlockStore;
unique_ptr<VersionCountingBlockStore> blockStore;
InMemoryBlockStore2 *baseBlockStore;
unique_ptr<VersionCountingBlockStore2> blockStore;
std::tie(baseBlockStore, blockStore) = makeBlockStoreWithoutDeletionPrevention();
auto key = blockStore->create(Data(0))->key();
auto key = blockStore->create(Data(0));
baseBlockStore->remove(key);
EXPECT_EQ(boost::none, blockStore->load(key));
}
// Check that in a single-client scenario, missing blocks are integrity errors.
TEST_F(VersionCountingBlockStoreTest, DeletionPrevention_DoesntAllowDeletingBlocksWhenActivated) {
FakeBlockStore *baseBlockStore;
unique_ptr<VersionCountingBlockStore> blockStore;
InMemoryBlockStore2 *baseBlockStore;
unique_ptr<VersionCountingBlockStore2> blockStore;
std::tie(baseBlockStore, blockStore) = makeBlockStoreWithDeletionPrevention();
auto key = blockStore->create(Data(0))->key();
auto key = blockStore->create(Data(0));
baseBlockStore->remove(key);
EXPECT_THROW(
blockStore->load(key),
@ -217,10 +209,10 @@ TEST_F(VersionCountingBlockStoreTest, DeletionPrevention_DoesntAllowDeletingBloc
// Check that in a multi-client scenario, missing blocks are not integrity errors, because another client might have deleted them.
TEST_F(VersionCountingBlockStoreTest, DeletionPrevention_InForEachBlock_AllowsDeletingBlocksWhenDeactivated) {
FakeBlockStore *baseBlockStore;
unique_ptr<VersionCountingBlockStore> blockStore;
InMemoryBlockStore2 *baseBlockStore;
unique_ptr<VersionCountingBlockStore2> blockStore;
std::tie(baseBlockStore, blockStore) = makeBlockStoreWithoutDeletionPrevention();
auto key = blockStore->create(Data(0))->key();
auto key = blockStore->create(Data(0));
baseBlockStore->remove(key);
int count = 0;
blockStore->forEachBlock([&count] (const blockstore::Key &) {
@ -231,10 +223,10 @@ TEST_F(VersionCountingBlockStoreTest, DeletionPrevention_InForEachBlock_AllowsDe
// Check that in a single-client scenario, missing blocks are integrity errors.
TEST_F(VersionCountingBlockStoreTest, DeletionPrevention_InForEachBlock_DoesntAllowDeletingBlocksWhenActivated) {
FakeBlockStore *baseBlockStore;
unique_ptr<VersionCountingBlockStore> blockStore;
InMemoryBlockStore2 *baseBlockStore;
unique_ptr<VersionCountingBlockStore2> blockStore;
std::tie(baseBlockStore, blockStore) = makeBlockStoreWithDeletionPrevention();
auto key = blockStore->create(Data(0))->key();
auto key = blockStore->create(Data(0));
baseBlockStore->remove(key);
EXPECT_THROW(
blockStore->forEachBlock([] (const blockstore::Key &) {}),
@ -254,13 +246,13 @@ TEST_F(VersionCountingBlockStoreTest, PhysicalBlockSize_zerophysical) {
TEST_F(VersionCountingBlockStoreTest, PhysicalBlockSize_zerovirtual) {
auto key = CreateBlockReturnKey(Data(0));
auto base = baseBlockStore->load(key).value();
EXPECT_EQ(0u, blockStore->blockSizeFromPhysicalBlockSize(base->size()));
EXPECT_EQ(0u, blockStore->blockSizeFromPhysicalBlockSize(base.size()));
}
TEST_F(VersionCountingBlockStoreTest, PhysicalBlockSize_negativeboundaries) {
// This tests that a potential if/else in blockSizeFromPhysicalBlockSize that catches negative values has the
// correct boundary set. We test the highest value that is negative and the smallest value that is positive.
auto physicalSizeForVirtualSizeZero = baseBlockStore->load(CreateBlockReturnKey(Data(0))).value()->size();
auto physicalSizeForVirtualSizeZero = baseBlockStore->load(CreateBlockReturnKey(Data(0))).value().size();
if (physicalSizeForVirtualSizeZero > 0) {
EXPECT_EQ(0u, blockStore->blockSizeFromPhysicalBlockSize(physicalSizeForVirtualSizeZero - 1));
}
@ -271,5 +263,5 @@ TEST_F(VersionCountingBlockStoreTest, PhysicalBlockSize_negativeboundaries) {
TEST_F(VersionCountingBlockStoreTest, PhysicalBlockSize_positive) {
auto key = CreateBlockReturnKey(Data(10*1024));
auto base = baseBlockStore->load(key).value();
EXPECT_EQ(10*1024u, blockStore->blockSizeFromPhysicalBlockSize(base->size()));
EXPECT_EQ(10*1024u, blockStore->blockSizeFromPhysicalBlockSize(base.size()));
}

178
test/blockstore/testutils/BlockStore2Test.h
View File

@ -56,146 +56,146 @@ public:
TYPED_TEST_CASE_P(BlockStore2Test);
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenCallingTryCreateOnExistingBlock_thenFails) {
blockstore::Key key = this->blockStore->create(cpputils::Data(1024)).get();
EXPECT_FALSE(this->blockStore->tryCreate(key, cpputils::Data(1024)).get());
blockstore::Key key = this->blockStore->create(cpputils::Data(1024));
EXPECT_FALSE(this->blockStore->tryCreate(key, cpputils::Data(1024)));
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenCallingTryCreateOnNonExistingBlock_thenSucceeds) {
blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
EXPECT_TRUE(this->blockStore->tryCreate(key, cpputils::Data(1024)).get());
EXPECT_TRUE(this->blockStore->tryCreate(key, cpputils::Data(1024)));
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenCallingTryCreateOnNonExistingBlock_thenSucceeds) {
this->blockStore->create(cpputils::Data(512));
blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
EXPECT_TRUE(this->blockStore->tryCreate(key, cpputils::Data(1024)).get());
EXPECT_TRUE(this->blockStore->tryCreate(key, cpputils::Data(1024)));
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenLoadExistingBlock_thenSucceeds) {
blockstore::Key key = this->blockStore->create(cpputils::Data(1024)).get();
EXPECT_NE(boost::none, this->blockStore->load(key).get());
blockstore::Key key = this->blockStore->create(cpputils::Data(1024));
EXPECT_NE(boost::none, this->blockStore->load(key));
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenLoadNonexistingBlock_thenFails) {
const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
EXPECT_EQ(boost::none, this->blockStore->load(key).get());
EXPECT_EQ(boost::none, this->blockStore->load(key));
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenLoadNonexistingBlock_thenFails) {
this->blockStore->create(cpputils::Data(512));
const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
EXPECT_EQ(boost::none, this->blockStore->load(key).get());
EXPECT_EQ(boost::none, this->blockStore->load(key));
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenStoringExistingBlock_thenSucceeds) {
blockstore::Key key = this->blockStore->create(cpputils::Data(1024)).get();
this->blockStore->store(key, cpputils::Data(1024)).wait();
blockstore::Key key = this->blockStore->create(cpputils::Data(1024));
this->blockStore->store(key, cpputils::Data(1024));
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenStoringNonexistingBlock_thenSucceeds) {
const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
this->blockStore->store(key, cpputils::Data(1024)).wait();
this->blockStore->store(key, cpputils::Data(1024));
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenStoringNonexistingBlock_thenSucceeds) {
this->blockStore->create(cpputils::Data(512));
const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
this->blockStore->store(key, cpputils::Data(1024)).wait();
this->blockStore->store(key, cpputils::Data(1024));
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenCreatingTwoBlocks_thenTheyGetDifferentKeys) {
blockstore::Key key1 = this->blockStore->create(cpputils::Data(1024)).get();
blockstore::Key key2 = this->blockStore->create(cpputils::Data(1024)).get();
blockstore::Key key1 = this->blockStore->create(cpputils::Data(1024));
blockstore::Key key2 = this->blockStore->create(cpputils::Data(1024));
EXPECT_NE(key1, key2);
}
TYPED_TEST_P(BlockStore2Test, givenOtherwiseEmptyBlockStore_whenRemovingBlock_thenBlockIsNotLoadableAnymore) {
blockstore::Key key = this->blockStore->create(cpputils::Data(1024)).get();
EXPECT_NE(boost::none, this->blockStore->load(key).get());
this->blockStore->remove(key).get();
EXPECT_EQ(boost::none, this->blockStore->load(key).get());
blockstore::Key key = this->blockStore->create(cpputils::Data(1024));
EXPECT_NE(boost::none, this->blockStore->load(key));
EXPECT_TRUE(this->blockStore->remove(key));
EXPECT_EQ(boost::none, this->blockStore->load(key));
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenRemovingBlock_thenBlockIsNotLoadableAnymore) {
blockstore::Key key = this->blockStore->create(cpputils::Data(1024)).get();
blockstore::Key key = this->blockStore->create(cpputils::Data(1024));
this->blockStore->create(cpputils::Data(512));
EXPECT_NE(boost::none, this->blockStore->load(key).get());
this->blockStore->remove(key).get();
EXPECT_EQ(boost::none, this->blockStore->load(key).get());
EXPECT_NE(boost::none, this->blockStore->load(key));
EXPECT_TRUE(this->blockStore->remove(key));
EXPECT_EQ(boost::none, this->blockStore->load(key));
}
TYPED_TEST_P(BlockStore2Test, givenOtherwiseEmptyBlockStore_whenRemovingExistingBlock_thenSucceeds) {
blockstore::Key key = this->blockStore->create(cpputils::Data(1024)).get();
EXPECT_EQ(true, this->blockStore->remove(key).get());
blockstore::Key key = this->blockStore->create(cpputils::Data(1024));
EXPECT_TRUE(this->blockStore->remove(key));
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenRemovingExistingBlock_thenSucceeds) {
blockstore::Key key = this->blockStore->create(cpputils::Data(1024)).get();
blockstore::Key key = this->blockStore->create(cpputils::Data(1024));
this->blockStore->create(cpputils::Data(512));
EXPECT_EQ(true, this->blockStore->remove(key).get());
EXPECT_EQ(true, this->blockStore->remove(key));
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenRemovingNonexistingBlock_thenFails) {
const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
auto result = this->blockStore->remove(key).get();
auto result = this->blockStore->remove(key);
EXPECT_EQ(false, result);
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenRemovingNonexistingBlock_thenFails) {
blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772973");
blockstore::Key differentKey = blockstore::Key::FromString("290AC2C7097274A389EE14B91B72B493");
ASSERT_TRUE(this->blockStore->tryCreate(key, cpputils::Data(1024)).get());
EXPECT_EQ(false, this->blockStore->remove(differentKey).get());
ASSERT_TRUE(this->blockStore->tryCreate(key, cpputils::Data(1024)));
EXPECT_EQ(false, this->blockStore->remove(differentKey));
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenCreatingAndLoadingEmptyBlock_thenCorrectBlockLoads) {
auto key = this->blockStore->create(cpputils::Data(0)).get();
auto loaded = this->blockStore->load(key).get().value();
auto key = this->blockStore->create(cpputils::Data(0));
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(0u, loaded.size());
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenCreatingAndLoadingEmptyBlock_thenCorrectBlockLoads) {
this->blockStore->create(cpputils::Data(512));
auto key = this->blockStore->create(cpputils::Data(0)).get();
auto loaded = this->blockStore->load(key).get().value();
auto key = this->blockStore->create(cpputils::Data(0));
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(0u, loaded.size());
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenCreatingAndLoadingNonEmptyBlock_thenCorrectBlockLoads) {
cpputils::Data data = cpputils::DataFixture::generate(1024);
auto key = this->blockStore->create(data.copy()).get();
auto loaded = this->blockStore->load(key).get().value();
auto key = this->blockStore->create(data.copy());
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(loaded, data);
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenCreatingAndLoadingNonEmptyBlock_thenCorrectBlockLoads) {
this->blockStore->create(cpputils::Data(512));
cpputils::Data data = cpputils::DataFixture::generate(1024);
auto key = this->blockStore->create(data.copy()).get();
auto loaded = this->blockStore->load(key).get().value();
auto key = this->blockStore->create(data.copy());
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(loaded, data);
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenTryCreatingAndLoadingEmptyBlock_thenCorrectBlockLoads) {
blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772973");
ASSERT_TRUE(this->blockStore->tryCreate(key, cpputils::Data(0)).get());
auto loaded = this->blockStore->load(key).get().value();
ASSERT_TRUE(this->blockStore->tryCreate(key, cpputils::Data(0)));
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(0u, loaded.size());
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenTryCreatingAndLoadingEmptyBlock_thenCorrectBlockLoads) {
blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772973");
this->blockStore->create(cpputils::Data(512));
ASSERT_TRUE(this->blockStore->tryCreate(key, cpputils::Data(0)).get());
auto loaded = this->blockStore->load(key).get().value();
ASSERT_TRUE(this->blockStore->tryCreate(key, cpputils::Data(0)));
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(0u, loaded.size());
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenTryCreatingAndLoadingNonEmptyBlock_thenCorrectBlockLoads) {
blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772973");
cpputils::Data data = cpputils::DataFixture::generate(1024);
ASSERT_TRUE(this->blockStore->tryCreate(key, data.copy()).get());
auto loaded = this->blockStore->load(key).get().value();
ASSERT_TRUE(this->blockStore->tryCreate(key, data.copy()));
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(loaded, data);
}
@ -203,31 +203,31 @@ TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenTryCreatingAndLoadingN
blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772973");
this->blockStore->create(cpputils::Data(512));
cpputils::Data data = cpputils::DataFixture::generate(1024);
ASSERT_TRUE(this->blockStore->tryCreate(key, data.copy()).get());
auto loaded = this->blockStore->load(key).get().value();
ASSERT_TRUE(this->blockStore->tryCreate(key, data.copy()));
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(loaded, data);
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenStoringAndLoadingNonExistingEmptyBlock_thenCorrectBlockLoads) {
const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
this->blockStore->store(key, cpputils::Data(0)).wait();
auto loaded = this->blockStore->load(key).get().value();
this->blockStore->store(key, cpputils::Data(0));
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(0u, loaded.size());
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenStoringAndLoadingNonExistingEmptyBlock_thenCorrectBlockLoads) {
this->blockStore->create(cpputils::Data(512));
const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
this->blockStore->store(key, cpputils::Data(0)).wait();
auto loaded = this->blockStore->load(key).get().value();
this->blockStore->store(key, cpputils::Data(0));
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(0u, loaded.size());
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenStoringAndLoadingNonExistingNonEmptyBlock_thenCorrectBlockLoads) {
const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
cpputils::Data data = cpputils::DataFixture::generate(1024);
this->blockStore->store(key, data.copy()).wait();
auto loaded = this->blockStore->load(key).get().value();
this->blockStore->store(key, data.copy());
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(data, loaded);
}
@ -235,52 +235,52 @@ TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenStoringAndLoadingNonEx
this->blockStore->create(cpputils::Data(512));
const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
cpputils::Data data = cpputils::DataFixture::generate(1024);
this->blockStore->store(key, data.copy()).wait();
auto loaded = this->blockStore->load(key).get().value();
this->blockStore->store(key, data.copy());
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(data, loaded);
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenStoringAndLoadingExistingEmptyBlock_thenCorrectBlockLoads) {
auto key = this->blockStore->create(cpputils::Data(512)).get();
this->blockStore->store(key, cpputils::Data(0)).wait();
auto loaded = this->blockStore->load(key).get().value();
auto key = this->blockStore->create(cpputils::Data(512));
this->blockStore->store(key, cpputils::Data(0));
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(0u, loaded.size());
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenStoringAndLoadingExistingEmptyBlock_thenCorrectBlockLoads) {
this->blockStore->create(cpputils::Data(512)).get();
auto key = this->blockStore->create(cpputils::Data(512)).get();
this->blockStore->store(key, cpputils::Data(0)).wait();
auto loaded = this->blockStore->load(key).get().value();
this->blockStore->create(cpputils::Data(512));
auto key = this->blockStore->create(cpputils::Data(512));
this->blockStore->store(key, cpputils::Data(0));
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(0u, loaded.size());
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenStoringAndLoadingExistingNonEmptyBlock_thenCorrectBlockLoads) {
auto key = this->blockStore->create(cpputils::Data(512)).get();
auto key = this->blockStore->create(cpputils::Data(512));
cpputils::Data data = cpputils::DataFixture::generate(1024);
this->blockStore->store(key, data.copy()).wait();
auto loaded = this->blockStore->load(key).get().value();
this->blockStore->store(key, data.copy());
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(data, loaded);
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenStoringAndLoadingExistingNonEmptyBlock_thenCorrectBlockLoads) {
this->blockStore->create(cpputils::Data(512)).get();
auto key = this->blockStore->create(cpputils::Data(512)).get();
this->blockStore->create(cpputils::Data(512));
auto key = this->blockStore->create(cpputils::Data(512));
cpputils::Data data = cpputils::DataFixture::generate(1024);
this->blockStore->store(key, data.copy()).wait();
auto loaded = this->blockStore->load(key).get().value();
this->blockStore->store(key, data.copy());
auto loaded = this->blockStore->load(key).value();
EXPECT_EQ(data, loaded);
}
TYPED_TEST_P(BlockStore2Test, givenEmptyBlockStore_whenLoadingNonExistingBlock_thenFails) {
const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
EXPECT_EQ(boost::none, this->blockStore->load(key).get());
EXPECT_EQ(boost::none, this->blockStore->load(key));
}
TYPED_TEST_P(BlockStore2Test, givenNonEmptyBlockStore_whenLoadingNonExistingBlock_thenFails) {
this->blockStore->create(cpputils::Data(512)).get();
this->blockStore->create(cpputils::Data(512));
const blockstore::Key key = blockstore::Key::FromString("1491BB4932A389EE14BC7090AC772972");
EXPECT_EQ(boost::none, this->blockStore->load(key).get());
EXPECT_EQ(boost::none, this->blockStore->load(key));
}
TYPED_TEST_P(BlockStore2Test, NumBlocksIsCorrectOnEmptyBlockstore) {
@ -290,29 +290,29 @@ TYPED_TEST_P(BlockStore2Test, NumBlocksIsCorrectOnEmptyBlockstore) {
TYPED_TEST_P(BlockStore2Test, NumBlocksIsCorrectAfterAddingOneBlock) {
auto blockStore = this->fixture.createBlockStore();
blockStore->create(cpputils::Data(1)).wait();
blockStore->create(cpputils::Data(1));
EXPECT_EQ(1u, blockStore->numBlocks());
}
TYPED_TEST_P(BlockStore2Test, NumBlocksIsCorrectAfterRemovingTheLastBlock) {
auto blockStore = this->fixture.createBlockStore();
blockstore::Key key = blockStore->create(cpputils::Data(1)).get();
blockStore->remove(key).wait();
blockstore::Key key = blockStore->create(cpputils::Data(1));
EXPECT_TRUE(blockStore->remove(key));
EXPECT_EQ(0u, blockStore->numBlocks());
}
TYPED_TEST_P(BlockStore2Test, NumBlocksIsCorrectAfterAddingTwoBlocks) {
auto blockStore = this->fixture.createBlockStore();
blockStore->create(cpputils::Data(1)).wait();
blockStore->create(cpputils::Data(0)).wait();
blockStore->create(cpputils::Data(1));
blockStore->create(cpputils::Data(0));
EXPECT_EQ(2u, blockStore->numBlocks());
}
TYPED_TEST_P(BlockStore2Test, NumBlocksIsCorrectAfterRemovingABlock) {
auto blockStore = this->fixture.createBlockStore();
blockstore::Key key = blockStore->create(cpputils::Data(1)).get();
blockStore->create(cpputils::Data(1)).wait();
blockStore->remove(key).wait();
blockstore::Key key = blockStore->create(cpputils::Data(1));
blockStore->create(cpputils::Data(1));
EXPECT_TRUE(blockStore->remove(key));
EXPECT_EQ(1u, blockStore->numBlocks());
}
@ -325,7 +325,7 @@ TYPED_TEST_P(BlockStore2Test, ForEachBlock_zeroblocks) {
TYPED_TEST_P(BlockStore2Test, ForEachBlock_oneblock) {
auto blockStore = this->fixture.createBlockStore();
auto key = blockStore->create(cpputils::Data(1)).get();
auto key = blockStore->create(cpputils::Data(1));
MockForEachBlockCallback mockForEachBlockCallback;
blockStore->forEachBlock(mockForEachBlockCallback.callback());
this->EXPECT_UNORDERED_EQ({key}, mockForEachBlockCallback.called_with);
@ -333,8 +333,8 @@ TYPED_TEST_P(BlockStore2Test, ForEachBlock_oneblock) {
TYPED_TEST_P(BlockStore2Test, ForEachBlock_twoblocks) {
auto blockStore = this->fixture.createBlockStore();
auto key1 = blockStore->create(cpputils::Data(1)).get();
auto key2 = blockStore->create(cpputils::Data(1)).get();
auto key1 = blockStore->create(cpputils::Data(1));
auto key2 = blockStore->create(cpputils::Data(1));
MockForEachBlockCallback mockForEachBlockCallback;
blockStore->forEachBlock(mockForEachBlockCallback.callback());
this->EXPECT_UNORDERED_EQ({key1, key2}, mockForEachBlockCallback.called_with);
@ -342,9 +342,9 @@ TYPED_TEST_P(BlockStore2Test, ForEachBlock_twoblocks) {
TYPED_TEST_P(BlockStore2Test, ForEachBlock_threeblocks) {
auto blockStore = this->fixture.createBlockStore();
auto key1 = blockStore->create(cpputils::Data(1)).get();
auto key2 = blockStore->create(cpputils::Data(1)).get();
auto key3 = blockStore->create(cpputils::Data(1)).get();
auto key1 = blockStore->create(cpputils::Data(1));
auto key2 = blockStore->create(cpputils::Data(1));
auto key3 = blockStore->create(cpputils::Data(1));
MockForEachBlockCallback mockForEachBlockCallback;
blockStore->forEachBlock(mockForEachBlockCallback.callback());
this->EXPECT_UNORDERED_EQ({key1, key2, key3}, mockForEachBlockCallback.called_with);
@ -352,8 +352,8 @@ TYPED_TEST_P(BlockStore2Test, ForEachBlock_threeblocks) {
TYPED_TEST_P(BlockStore2Test, ForEachBlock_doesntListRemovedBlocks_oneblock) {
auto blockStore = this->fixture.createBlockStore();
auto key1 = blockStore->create(cpputils::Data(1)).get();
blockStore->remove(key1).wait();
auto key1 = blockStore->create(cpputils::Data(1));
EXPECT_TRUE(blockStore->remove(key1));
MockForEachBlockCallback mockForEachBlockCallback;
blockStore->forEachBlock(mockForEachBlockCallback.callback());
this->EXPECT_UNORDERED_EQ({}, mockForEachBlockCallback.called_with);
@ -361,9 +361,9 @@ TYPED_TEST_P(BlockStore2Test, ForEachBlock_doesntListRemovedBlocks_oneblock) {
TYPED_TEST_P(BlockStore2Test, ForEachBlock_doesntListRemovedBlocks_twoblocks) {
auto blockStore = this->fixture.createBlockStore();
auto key1 = blockStore->create(cpputils::Data(1)).get();
auto key2 = blockStore->create(cpputils::Data(1)).get();
blockStore->remove(key1);
auto key1 = blockStore->create(cpputils::Data(1));
auto key2 = blockStore->create(cpputils::Data(1));
EXPECT_TRUE(blockStore->remove(key1));
MockForEachBlockCallback mockForEachBlockCallback;
blockStore->forEachBlock(mockForEachBlockCallback.callback());
this->EXPECT_UNORDERED_EQ({key2}, mockForEachBlockCallback.called_with);

2
test/blockstore/testutils/BlockStoreWithRandomKeysTest.h
View File

@ -9,7 +9,7 @@
class BlockStoreWithRandomKeysTestFixture {
public:
virtual ~BlockStoreWithRandomKeysTestFixture() {}
virtual cpputils::unique_ref<blockstore::BlockStoreWithRandomKeys> createBlockStore() = 0;
virtual cpputils::unique_ref<blockstore::BlockStore> createBlockStore() = 0;
};
template<class ConcreteBlockStoreWithRandomKeysTestFixture>

8
test/cryfs/config/CryCipherTest.cpp
View File

@ -53,7 +53,7 @@ public:
unique_ref<InMemoryBlockStore2> _baseStore = make_unique_ref<InMemoryBlockStore2>();
InMemoryBlockStore2 *baseStore = _baseStore.get();
unique_ref<BlockStore2> encryptedStore = cipher.createEncryptedBlockstore(std::move(_baseStore), encKey);
bool created = encryptedStore->tryCreate(key, std::move(data)).get();
bool created = encryptedStore->tryCreate(key, std::move(data));
EXPECT_TRUE(created);
return _loadBlock(baseStore, key);
}
@ -61,14 +61,14 @@ public:
template<class Cipher>
Data _decryptUsingEncryptedBlockStoreWithCipher(const std::string &encKey, const blockstore::Key &key, Data data) {
unique_ref<InMemoryBlockStore2> baseStore = make_unique_ref<InMemoryBlockStore2>();
bool created = baseStore->tryCreate(key, std::move(data)).get();
bool created = baseStore->tryCreate(key, std::move(data));
EXPECT_TRUE(created);
EncryptedBlockStore2<Cipher> encryptedStore(std::move(baseStore), Cipher::EncryptionKey::FromString(encKey));
return _loadBlock(&encryptedStore, key);
}
Data _loadBlock(BlockStore2 *store, const blockstore::Key &key) {
return store->load(key).get().value();
return store->load(key).value();
}
};
@ -117,7 +117,7 @@ TEST_F(CryCipherTest, ThereIsACipherWithoutWarning) {
}
TEST_F(CryCipherTest, ThereIsACipherWithIntegrityWarning) {
EXPECT_THAT(CryCiphers::find("aes-256-cfb").warning().get(), MatchesRegex(".*integrity.*"));
EXPECT_THAT(CryCiphers::find("aes-256-cfb").warning().value(), MatchesRegex(".*integrity.*"));
}
#if CRYPTOPP_VERSION != 564