From 03597b94744a25fe275a0adc4cacfdf6941bb2e7 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 24 Apr 2015 13:42:37 +0200 Subject: [PATCH 01/41] Code formatting & TODOs --- implementations/caching/Cache.h | 1 - implementations/caching/CachingBlockStore.cpp | 1 - implementations/caching/CachingBlockStore.h | 1 - interface/BlockStore.h | 12 ++++++------ 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/implementations/caching/Cache.h b/implementations/caching/Cache.h index 7839b086..740e9003 100644 --- a/implementations/caching/Cache.h +++ b/implementations/caching/Cache.h @@ -13,7 +13,6 @@ namespace caching { class PeriodicTask; //TODO Test -//TODO Also throw blocks out after a timeout class Cache { public: diff --git a/implementations/caching/CachingBlockStore.cpp b/implementations/caching/CachingBlockStore.cpp index 0e152f37..532e32c6 100644 --- a/implementations/caching/CachingBlockStore.cpp +++ b/implementations/caching/CachingBlockStore.cpp @@ -52,7 +52,6 @@ void CachingBlockStore::remove(std::unique_ptr block) { } uint64_t CachingBlockStore::numBlocks() const { - //TODO Add number of NewBlock instances return _baseBlockStore->numBlocks() + _numNewBlocks; } diff --git a/implementations/caching/CachingBlockStore.h b/implementations/caching/CachingBlockStore.h index bf9adbc5..49c67c27 100644 --- a/implementations/caching/CachingBlockStore.h +++ b/implementations/caching/CachingBlockStore.h @@ -8,7 +8,6 @@ namespace blockstore { namespace caching { -//TODO Rename this to CachingBlockStore and the other one to something else //TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel) class CachingBlockStore: public BlockStore { public: diff --git a/interface/BlockStore.h b/interface/BlockStore.h index 8ea14ed3..74d8d59d 100644 --- a/interface/BlockStore.h +++ b/interface/BlockStore.h @@ -23,12 +23,12 @@ public: virtual uint64_t numBlocks() const = 0; std::unique_ptr create(const Data &data) { - std::unique_ptr block(nullptr); - while(block.get() == nullptr) { - //TODO Copy necessary? - block = tryCreate(createKey(), data.copy()); - } - return block; + std::unique_ptr block(nullptr); + while(block.get() == nullptr) { + //TODO Copy necessary? + block = tryCreate(createKey(), data.copy()); + } + return block; } }; From 6e50c9d8a552ac39838f097065ad5c73c87b071d Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 24 Apr 2015 18:14:25 +0200 Subject: [PATCH 02/41] Factor out encryption algorithm from EncryptedBlockStore --- implementations/encrypted/Cipher.cpp | 2 + implementations/encrypted/Cipher.h | 26 ++++ implementations/encrypted/EncryptedBlock.cpp | 90 -------------- implementations/encrypted/EncryptedBlock.h | 113 ++++++++++++++---- .../encrypted/EncryptedBlockStore.cpp | 41 ------- .../encrypted/EncryptedBlockStore.h | 46 ++++++- implementations/encrypted/EncryptionKey.cpp | 1 - implementations/encrypted/EncryptionKey.h | 15 --- .../encrypted/ciphers/AES256_CFB.cpp | 27 +++++ .../encrypted/ciphers/AES256_CFB.h | 36 ++++++ .../encrypted/EncryptedBlockStoreTest.cpp | 5 +- utils/FixedSizeData.h | 28 ++--- 12 files changed, 242 insertions(+), 188 deletions(-) create mode 100644 implementations/encrypted/Cipher.cpp create mode 100644 implementations/encrypted/Cipher.h delete mode 100644 implementations/encrypted/EncryptionKey.cpp delete mode 100644 implementations/encrypted/EncryptionKey.h create mode 100644 implementations/encrypted/ciphers/AES256_CFB.cpp create mode 100644 implementations/encrypted/ciphers/AES256_CFB.h diff --git a/implementations/encrypted/Cipher.cpp b/implementations/encrypted/Cipher.cpp new file mode 100644 index 00000000..c866c29b --- /dev/null +++ b/implementations/encrypted/Cipher.cpp @@ -0,0 +1,2 @@ +#include "Cipher.h" + diff --git a/implementations/encrypted/Cipher.h b/implementations/encrypted/Cipher.h new file mode 100644 index 00000000..b5528a34 --- /dev/null +++ b/implementations/encrypted/Cipher.h @@ -0,0 +1,26 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHER_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHER_H_ + +#include + +namespace blockstore { +namespace encrypted { + +class Cipher { +public: + virtual ~Cipher() {} + + virtual unsigned int ciphertextBlockSize(unsigned int ciphertextBlockSize) const = 0; + virtual unsigned int plaintextBlockSize(unsigned int plaintextBlockSize) const = 0; + + virtual void encrypt(const byte *plaintext, unsigned int plaintextSize, byte *ciphertext) const = 0; + virtual void decrypt(const byte *ciphertext, byte *plaintext, unsigned int plaintextSize) const = 0; +}; + +} +} + + + +#endif diff --git a/implementations/encrypted/EncryptedBlock.cpp b/implementations/encrypted/EncryptedBlock.cpp index 59da307d..5f66d9c9 100644 --- a/implementations/encrypted/EncryptedBlock.cpp +++ b/implementations/encrypted/EncryptedBlock.cpp @@ -1,91 +1 @@ #include "EncryptedBlock.h" -#include - -#include "../../utils/BlockStoreUtils.h" - -using CryptoPP::CFB_Mode; -using CryptoPP::AES; - -using std::make_unique; - -//TODO not only encryption, but also hmac - -namespace blockstore { -namespace encrypted { - -constexpr unsigned int EncryptedBlock::IV_SIZE; - -std::unique_ptr EncryptedBlock::TryCreateNew(BlockStore *baseBlockStore, const Key &key, Data data, const EncryptionKey &encKey) { - Data encrypted = _encrypt(data, encKey); - auto baseBlock = baseBlockStore->tryCreate(key, std::move(encrypted)); - if (baseBlock.get() == nullptr) { - //TODO Test this code branch - return nullptr; - } - - return make_unique(std::move(baseBlock), encKey, std::move(data)); -} - -EncryptedBlock::EncryptedBlock(std::unique_ptr baseBlock, const EncryptionKey &encKey) - :EncryptedBlock(std::move(baseBlock), encKey, Data(USEABLE_BLOCK_SIZE(baseBlock->size()))) { - _decryptFromBaseBlock(); -} - -EncryptedBlock::EncryptedBlock(std::unique_ptr baseBlock, const EncryptionKey &encKey, Data plaintextData) - :Block(baseBlock->key()), - _baseBlock(std::move(baseBlock)), - _plaintextData(std::move(plaintextData)), - _encKey(encKey), - _dataChanged(false) { -} - -EncryptedBlock::~EncryptedBlock() { - _encryptToBaseBlock(); -} - -const void *EncryptedBlock::data() const { - return _plaintextData.data(); -} - -void EncryptedBlock::write(const void *source, uint64_t offset, uint64_t size) { - assert(offset <= _plaintextData.size() && offset + size <= _plaintextData.size()); //Also check offset < _data->size() because of possible overflow in the addition - std::memcpy((uint8_t*)_plaintextData.data()+offset, source, size); - _dataChanged = true; -} - -void EncryptedBlock::flush() { - _encryptToBaseBlock(); - return _baseBlock->flush(); -} - -size_t EncryptedBlock::size() const { - return _plaintextData.size(); -} - -void EncryptedBlock::_decryptFromBaseBlock() { - const byte *iv = (byte*)_baseBlock->data(); - const byte *data = (byte*)_baseBlock->data() + IV_SIZE; - auto decryption = CFB_Mode::Decryption((byte*)_encKey.data(), EncryptionKey::BINARY_LENGTH, iv); - decryption.ProcessData((byte*)_plaintextData.data(), data, _plaintextData.size()); -} - -void EncryptedBlock::_encryptToBaseBlock() { - if (_dataChanged) { - Data encrypted = _encrypt(_plaintextData, _encKey); - _baseBlock->write(encrypted.data(), 0, encrypted.size()); - _dataChanged = false; - } -} - -Data EncryptedBlock::_encrypt(const Data &plaintext, const EncryptionKey &encKey) { - FixedSizeData iv = FixedSizeData::CreateRandom(); - auto encryption = CFB_Mode::Encryption(encKey.data(), EncryptionKey::BINARY_LENGTH, iv.data()); - //TODO More performance when not using "Data encrypted" object, but encrypting directly to a target that was specified via a parameter using a specialized CryptoPP sink - Data encrypted(IV_SIZE + plaintext.size()); - std::memcpy(encrypted.data(), iv.data(), IV_SIZE); - encryption.ProcessData((byte*)encrypted.data() + IV_SIZE, (byte*)plaintext.data(), plaintext.size()); - return encrypted; -} - -} -} diff --git a/implementations/encrypted/EncryptedBlock.h b/implementations/encrypted/EncryptedBlock.h index 84b66019..4be10ef9 100644 --- a/implementations/encrypted/EncryptedBlock.h +++ b/implementations/encrypted/EncryptedBlock.h @@ -3,7 +3,7 @@ #define BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_ #include "../../interface/Block.h" -#include "EncryptionKey.h" +#include "Cipher.h" #include "../../utils/Data.h" #include "../../interface/BlockStore.h" @@ -12,16 +12,19 @@ namespace blockstore { namespace encrypted { -class EncryptedBlockStore; +template class EncryptedBlockStore; +//TODO not only encryption, but also hmac + +template class EncryptedBlock: public Block { public: - //TODO Storing key twice (in parent class and in object pointed to). Once would be enough. - EncryptedBlock(std::unique_ptr baseBlock, const EncryptionKey &encKey); - EncryptedBlock(std::unique_ptr baseBlock, const EncryptionKey &encKey, Data plaintextData); - virtual ~EncryptedBlock(); + static std::unique_ptr TryCreateNew(BlockStore *baseBlockStore, const Key &key, Data data, const typename Cipher::EncryptionKey &encKey); - static std::unique_ptr TryCreateNew(BlockStore *baseBlockStore, const Key &key, Data data, const EncryptionKey &encKey); + //TODO Storing key twice (in parent class and in object pointed to). Once would be enough. + EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key); + EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key, Data plaintextData); + virtual ~EncryptedBlock(); const void *data() const override; void write(const void *source, uint64_t offset, uint64_t size) override; @@ -29,33 +32,99 @@ public: size_t size() const override; - static constexpr unsigned int BASE_BLOCK_SIZE(unsigned int useableBlockSize) { - return useableBlockSize + IV_SIZE; - } - - static constexpr unsigned int USEABLE_BLOCK_SIZE(unsigned int baseBlockSize) { - return baseBlockSize - IV_SIZE; - } + std::unique_ptr releaseBlock(); private: std::unique_ptr _baseBlock; Data _plaintextData; - EncryptionKey _encKey; + typename Cipher::EncryptionKey _encKey; bool _dataChanged; - static constexpr unsigned int IV_SIZE = CryptoPP::AES::BLOCKSIZE; - - byte *baseBlockIV(); - byte *baseBlockData(); - void _encryptToBaseBlock(); void _decryptFromBaseBlock(); - static Data _encrypt(const Data &plaintext, const EncryptionKey &encKey); - DISALLOW_COPY_AND_ASSIGN(EncryptedBlock); }; + + +template +std::unique_ptr> EncryptedBlock::TryCreateNew(BlockStore *baseBlockStore, const Key &key, Data data, const typename Cipher::EncryptionKey &encKey) { + Data encrypted(Cipher::ciphertextSize(data.size())); + Cipher::encrypt((byte*)data.data(), data.size(), (byte*)encrypted.data(), encKey); + auto baseBlock = baseBlockStore->tryCreate(key, std::move(encrypted)); + if (baseBlock.get() == nullptr) { + //TODO Test this code branch + return nullptr; + } + + return std::make_unique(std::move(baseBlock), encKey, std::move(data)); +} + +template +EncryptedBlock::EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &encKey) + :EncryptedBlock(std::move(baseBlock), encKey, Data(Cipher::plaintextSize(baseBlock->size()))) { + _decryptFromBaseBlock(); +} + +template +EncryptedBlock::EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &encKey, Data plaintextData) + :Block(baseBlock->key()), + _baseBlock(std::move(baseBlock)), + _plaintextData(std::move(plaintextData)), + _encKey(encKey), + _dataChanged(false) { +} + +template +EncryptedBlock::~EncryptedBlock() { + _encryptToBaseBlock(); +} + +template +const void *EncryptedBlock::data() const { + return _plaintextData.data(); +} + +template +void EncryptedBlock::write(const void *source, uint64_t offset, uint64_t size) { + assert(offset <= _plaintextData.size() && offset + size <= _plaintextData.size()); //Also check offset < _data->size() because of possible overflow in the addition + std::memcpy((uint8_t*)_plaintextData.data()+offset, source, size); + _dataChanged = true; +} + +template +void EncryptedBlock::flush() { + _encryptToBaseBlock(); + return _baseBlock->flush(); +} + +template +size_t EncryptedBlock::size() const { + return _plaintextData.size(); +} + +template +void EncryptedBlock::_decryptFromBaseBlock() { + //TODO Change BlockStore so we can read their "class Data" objects instead of "void *data()", and then we can change the Cipher interface to take Data objects instead of "byte *" + size + Cipher::decrypt((byte*)_baseBlock->data(), (byte*)_plaintextData.data(), _plaintextData.size(), _encKey); +} + +template +void EncryptedBlock::_encryptToBaseBlock() { + if (_dataChanged) { + Data encrypted(Cipher::ciphertextSize(_plaintextData.size())); + Cipher::encrypt((byte*)_plaintextData.data(), _plaintextData.size(), (byte*)encrypted.data(), _encKey); + _baseBlock->write(encrypted.data(), 0, encrypted.size()); + _dataChanged = false; + } +} + +template +std::unique_ptr EncryptedBlock::releaseBlock() { + return std::move(_baseBlock); +} + } } diff --git a/implementations/encrypted/EncryptedBlockStore.cpp b/implementations/encrypted/EncryptedBlockStore.cpp index 1eb88f42..5ab7ca32 100644 --- a/implementations/encrypted/EncryptedBlockStore.cpp +++ b/implementations/encrypted/EncryptedBlockStore.cpp @@ -1,42 +1 @@ #include "EncryptedBlockStore.h" -#include "EncryptedBlock.h" -#include -#include "../../utils/BlockStoreUtils.h" - -using std::unique_ptr; -using std::make_unique; - -namespace blockstore { -namespace encrypted { - -EncryptedBlockStore::EncryptedBlockStore(unique_ptr baseBlockStore, const EncryptionKey &encKey) - : _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) { -} - -Key EncryptedBlockStore::createKey() { - return _baseBlockStore->createKey(); -} - -unique_ptr EncryptedBlockStore::tryCreate(const Key &key, Data data) { - return EncryptedBlock::TryCreateNew(_baseBlockStore.get(), key, std::move(data), _encKey); -} - -unique_ptr EncryptedBlockStore::load(const Key &key) { - auto block = _baseBlockStore->load(key); - if (block.get() == nullptr) { - return nullptr; - } - return make_unique(std::move(block), _encKey); -} - -void EncryptedBlockStore::remove(unique_ptr block) { - return _baseBlockStore->remove(std::move(block)); -} - -uint64_t EncryptedBlockStore::numBlocks() const { - return _baseBlockStore->numBlocks(); -} - -} -} - diff --git a/implementations/encrypted/EncryptedBlockStore.h b/implementations/encrypted/EncryptedBlockStore.h index 2e897d1d..c7933216 100644 --- a/implementations/encrypted/EncryptedBlockStore.h +++ b/implementations/encrypted/EncryptedBlockStore.h @@ -4,14 +4,17 @@ #include "../../interface/BlockStore.h" #include -#include "EncryptionKey.h" +#include +#include "Cipher.h" +#include "EncryptedBlock.h" namespace blockstore { namespace encrypted { +template class EncryptedBlockStore: public BlockStore { public: - EncryptedBlockStore(std::unique_ptr baseBlockStore, const EncryptionKey &encKey); + EncryptedBlockStore(std::unique_ptr baseBlockStore, const typename Cipher::EncryptionKey &encKey); Key createKey() override; std::unique_ptr tryCreate(const Key &key, Data data) override; @@ -21,11 +24,48 @@ public: private: std::unique_ptr _baseBlockStore; - EncryptionKey _encKey; + typename Cipher::EncryptionKey _encKey; DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore); }; + + +template +EncryptedBlockStore::EncryptedBlockStore(std::unique_ptr baseBlockStore, const typename Cipher::EncryptionKey &encKey) + : _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) { +} + +template +Key EncryptedBlockStore::createKey() { + return _baseBlockStore->createKey(); +} + +template +std::unique_ptr EncryptedBlockStore::tryCreate(const Key &key, Data data) { + return EncryptedBlock::TryCreateNew(_baseBlockStore.get(), key, std::move(data), _encKey); +} + +template +std::unique_ptr EncryptedBlockStore::load(const Key &key) { + auto block = _baseBlockStore->load(key); + if (block.get() == nullptr) { + return nullptr; + } + return std::make_unique>(std::move(block), _encKey); +} + +template +void EncryptedBlockStore::remove(std::unique_ptr block) { + auto baseBlock = cpputils::dynamic_pointer_move>(block)->releaseBlock(); + return _baseBlockStore->remove(std::move(baseBlock)); +} + +template +uint64_t EncryptedBlockStore::numBlocks() const { + return _baseBlockStore->numBlocks(); +} + } } diff --git a/implementations/encrypted/EncryptionKey.cpp b/implementations/encrypted/EncryptionKey.cpp deleted file mode 100644 index 18a97b45..00000000 --- a/implementations/encrypted/EncryptionKey.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "EncryptionKey.h" diff --git a/implementations/encrypted/EncryptionKey.h b/implementations/encrypted/EncryptionKey.h deleted file mode 100644 index 97cd7a9f..00000000 --- a/implementations/encrypted/EncryptionKey.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once -#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTIONKEY_H_ -#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTIONKEY_H_ - -#include "../../utils/FixedSizeData.h" - -namespace blockstore { -namespace encrypted { - -using EncryptionKey = FixedSizeData; - -} -} - -#endif diff --git a/implementations/encrypted/ciphers/AES256_CFB.cpp b/implementations/encrypted/ciphers/AES256_CFB.cpp new file mode 100644 index 00000000..bad5562f --- /dev/null +++ b/implementations/encrypted/ciphers/AES256_CFB.cpp @@ -0,0 +1,27 @@ +#include +#include "AES256_CFB.h" + +using CryptoPP::CFB_Mode; +using CryptoPP::AES; + +namespace blockstore { +namespace encrypted { + +constexpr unsigned int AES256_CFB::IV_SIZE; + +void AES256_CFB::encrypt(const byte *plaintext, unsigned int plaintextSize, byte *ciphertext, const EncryptionKey &encKey) { + FixedSizeData iv = FixedSizeData::CreateRandom(); + auto encryption = CFB_Mode::Encryption(encKey.data(), encKey.BINARY_LENGTH, iv.data()); + std::memcpy(ciphertext, iv.data(), IV_SIZE); + encryption.ProcessData(ciphertext + IV_SIZE, plaintext, plaintextSize); +} + +void AES256_CFB::decrypt(const byte *ciphertext, byte *plaintext, unsigned int plaintextSize, const EncryptionKey &encKey) { + const byte *iv = ciphertext; + const byte *data = ciphertext + IV_SIZE; + auto decryption = CFB_Mode::Decryption((byte*)encKey.data(), encKey.BINARY_LENGTH, iv); + decryption.ProcessData(plaintext, data, plaintextSize); +} + +} +} diff --git a/implementations/encrypted/ciphers/AES256_CFB.h b/implementations/encrypted/ciphers/AES256_CFB.h new file mode 100644 index 00000000..16c8e5f8 --- /dev/null +++ b/implementations/encrypted/ciphers/AES256_CFB.h @@ -0,0 +1,36 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHERS_AES256_CFB_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHERS_AES256_CFB_H_ + +#include "../../../utils/FixedSizeData.h" +#include + +namespace blockstore { +namespace encrypted { + +class AES256_CFB { +public: + using EncryptionKey = FixedSizeData<32>; + static_assert(32 == CryptoPP::AES::MAX_KEYLENGTH, "If AES offered larger keys, we should offer a variant with it"); + + AES256_CFB(const EncryptionKey &key); + + static constexpr unsigned int ciphertextSize(unsigned int plaintextBlockSize) { + return plaintextBlockSize + IV_SIZE; + } + + static constexpr unsigned int plaintextSize(unsigned int ciphertextBlockSize) { + return ciphertextBlockSize - IV_SIZE; + } + + static void encrypt(const byte *plaintext, unsigned int plaintextSize, byte *ciphertext, const EncryptionKey &key); + static void decrypt(const byte *ciphertext, byte *plaintext, unsigned int plaintextSize, const EncryptionKey &key); + +private: + static constexpr unsigned int IV_SIZE = CryptoPP::AES::BLOCKSIZE; +}; + +} +} + +#endif diff --git a/test/implementations/encrypted/EncryptedBlockStoreTest.cpp b/test/implementations/encrypted/EncryptedBlockStoreTest.cpp index b5c6eb29..4a8b35bb 100644 --- a/test/implementations/encrypted/EncryptedBlockStoreTest.cpp +++ b/test/implementations/encrypted/EncryptedBlockStoreTest.cpp @@ -1,3 +1,4 @@ +#include "../../../implementations/encrypted/ciphers/AES256_CFB.h" #include "../../../implementations/encrypted/EncryptedBlockStore.h" #include "../../../implementations/testfake/FakeBlockStore.h" #include "../../testutils/BlockStoreTest.h" @@ -6,8 +7,8 @@ using blockstore::BlockStore; using blockstore::encrypted::EncryptedBlockStore; -using blockstore::encrypted::EncryptionKey; using blockstore::testfake::FakeBlockStore; +using blockstore::encrypted::AES256_CFB; using std::unique_ptr; using std::make_unique; @@ -15,7 +16,7 @@ using std::make_unique; class EncryptedBlockStoreTestFixture: public BlockStoreTestFixture { public: unique_ptr createBlockStore() override { - return make_unique(make_unique(), EncryptionKey::FromString("1491BB4932A389EE14BC7090A272EE5517627CFA147A971A8E6E747E0C772972")); + return make_unique>(make_unique(), AES256_CFB::EncryptionKey::FromString("1491BB4932A389EE14BC7090A272EE5517627CFA147A971A8E6E747E0C772972")); } }; diff --git a/utils/FixedSizeData.h b/utils/FixedSizeData.h index f7b70f5c..1199275f 100644 --- a/utils/FixedSizeData.h +++ b/utils/FixedSizeData.h @@ -9,7 +9,7 @@ namespace blockstore { -template +template class FixedSizeData { public: //Non-virtual destructor because we want objects to be small @@ -35,28 +35,28 @@ private: unsigned char _data[BINARY_LENGTH]; }; -template bool operator==(const FixedSizeData &lhs, const FixedSizeData &rhs); -template bool operator!=(const FixedSizeData &lhs, const FixedSizeData &rhs); +template bool operator==(const FixedSizeData &lhs, const FixedSizeData &rhs); +template bool operator!=(const FixedSizeData &lhs, const FixedSizeData &rhs); // ----- Implementation ----- -template constexpr unsigned int FixedSizeData::BINARY_LENGTH; -template constexpr unsigned int FixedSizeData::STRING_LENGTH; +template constexpr unsigned int FixedSizeData::BINARY_LENGTH; +template constexpr unsigned int FixedSizeData::STRING_LENGTH; -template +template CryptoPP::AutoSeededRandomPool &FixedSizeData::RandomPool() { static CryptoPP::AutoSeededRandomPool singleton; return singleton; } -template +template FixedSizeData FixedSizeData::CreateRandom() { FixedSizeData result; RandomPool().GenerateBlock(result._data, BINARY_LENGTH); return result; } -template +template FixedSizeData FixedSizeData::FromString(const std::string &data) { assert(data.size() == STRING_LENGTH); FixedSizeData result; @@ -68,7 +68,7 @@ FixedSizeData FixedSizeData::FromString(const std::string &data) { return result; } -template +template std::string FixedSizeData::ToString() const { std::string result; CryptoPP::ArraySource(_data, BINARY_LENGTH, true, @@ -80,29 +80,29 @@ std::string FixedSizeData::ToString() const { return result; } -template +template const unsigned char *FixedSizeData::data() const { return _data; } -template +template void FixedSizeData::ToBinary(void *target) const { std::memcpy(target, _data, BINARY_LENGTH); } -template +template FixedSizeData FixedSizeData::FromBinary(const void *source) { FixedSizeData result; std::memcpy(result._data, source, BINARY_LENGTH); return result; } -template +template bool operator==(const FixedSizeData &lhs, const FixedSizeData &rhs) { return 0 == std::memcmp(lhs.data(), rhs.data(), FixedSizeData::BINARY_LENGTH); } -template +template bool operator!=(const FixedSizeData &lhs, const FixedSizeData &rhs) { return !operator==(lhs, rhs); } From f18195757c0520aedc1d7ab77763fdf55d4b4074 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 24 Apr 2015 18:58:42 +0200 Subject: [PATCH 03/41] - Added test cases for ciphers - Removed Cipher.h --- implementations/encrypted/Cipher.cpp | 2 - implementations/encrypted/Cipher.h | 26 --- implementations/encrypted/EncryptedBlock.h | 1 - .../encrypted/EncryptedBlockStore.h | 1 - .../encrypted/ciphers/AES256_CFB.h | 1 + test/implementations/encrypted/CipherTest.cpp | 155 ++++++++++++++++++ 6 files changed, 156 insertions(+), 30 deletions(-) delete mode 100644 implementations/encrypted/Cipher.cpp delete mode 100644 implementations/encrypted/Cipher.h create mode 100644 test/implementations/encrypted/CipherTest.cpp diff --git a/implementations/encrypted/Cipher.cpp b/implementations/encrypted/Cipher.cpp deleted file mode 100644 index c866c29b..00000000 --- a/implementations/encrypted/Cipher.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#include "Cipher.h" - diff --git a/implementations/encrypted/Cipher.h b/implementations/encrypted/Cipher.h deleted file mode 100644 index b5528a34..00000000 --- a/implementations/encrypted/Cipher.h +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once -#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHER_H_ -#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHER_H_ - -#include - -namespace blockstore { -namespace encrypted { - -class Cipher { -public: - virtual ~Cipher() {} - - virtual unsigned int ciphertextBlockSize(unsigned int ciphertextBlockSize) const = 0; - virtual unsigned int plaintextBlockSize(unsigned int plaintextBlockSize) const = 0; - - virtual void encrypt(const byte *plaintext, unsigned int plaintextSize, byte *ciphertext) const = 0; - virtual void decrypt(const byte *ciphertext, byte *plaintext, unsigned int plaintextSize) const = 0; -}; - -} -} - - - -#endif diff --git a/implementations/encrypted/EncryptedBlock.h b/implementations/encrypted/EncryptedBlock.h index 4be10ef9..ee64a56b 100644 --- a/implementations/encrypted/EncryptedBlock.h +++ b/implementations/encrypted/EncryptedBlock.h @@ -3,7 +3,6 @@ #define BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_ #include "../../interface/Block.h" -#include "Cipher.h" #include "../../utils/Data.h" #include "../../interface/BlockStore.h" diff --git a/implementations/encrypted/EncryptedBlockStore.h b/implementations/encrypted/EncryptedBlockStore.h index c7933216..a8ce51ac 100644 --- a/implementations/encrypted/EncryptedBlockStore.h +++ b/implementations/encrypted/EncryptedBlockStore.h @@ -5,7 +5,6 @@ #include "../../interface/BlockStore.h" #include #include -#include "Cipher.h" #include "EncryptedBlock.h" namespace blockstore { diff --git a/implementations/encrypted/ciphers/AES256_CFB.h b/implementations/encrypted/ciphers/AES256_CFB.h index 16c8e5f8..46d4aece 100644 --- a/implementations/encrypted/ciphers/AES256_CFB.h +++ b/implementations/encrypted/ciphers/AES256_CFB.h @@ -8,6 +8,7 @@ namespace blockstore { namespace encrypted { +//TODO Add contract/interface for ciphers class AES256_CFB { public: using EncryptionKey = FixedSizeData<32>; diff --git a/test/implementations/encrypted/CipherTest.cpp b/test/implementations/encrypted/CipherTest.cpp new file mode 100644 index 00000000..0f1429cf --- /dev/null +++ b/test/implementations/encrypted/CipherTest.cpp @@ -0,0 +1,155 @@ +#include +#include "../../../implementations/encrypted/ciphers/AES256_CFB.h" + +#include "../../testutils/DataBlockFixture.h" +#include "../../../utils/Data.h" + +using namespace blockstore::encrypted; +using blockstore::Data; + +template +class CipherTest: public ::testing::Test { +public: + typename Cipher::EncryptionKey encKey = createRandomKey(); + + static typename Cipher::EncryptionKey createRandomKey(int seed = 0) { + DataBlockFixture data(Cipher::EncryptionKey::BINARY_LENGTH, seed); + return Cipher::EncryptionKey::FromBinary(data.data()); + } + + void CheckEncryptThenDecryptIsIdentity(const Data &plaintext) { + Data ciphertext(Cipher::ciphertextSize(plaintext.size())); + Data decrypted(plaintext.size()); + Cipher::encrypt((byte*)plaintext.data(), plaintext.size(), (byte*)ciphertext.data(), this->encKey); + Cipher::decrypt((byte*)ciphertext.data(), (byte*) decrypted.data(), decrypted.size(), this->encKey); + EXPECT_EQ(0, std::memcmp(plaintext.data(), decrypted.data(), plaintext.size())); + } + + Data CreateZeroes(unsigned int size) { + Data zeroes(size); + zeroes.FillWithZeroes(); + return zeroes; + } + + Data CreateData(unsigned int size, unsigned int seed = 0) { + DataBlockFixture data(size, seed); + Data result(size); + std::memcpy(result.data(), data.data(), size); + return result; + } +}; + +TYPED_TEST_CASE_P(CipherTest); + +TYPED_TEST_P(CipherTest, Size_0) { + EXPECT_EQ(0, TypeParam::ciphertextSize(TypeParam::plaintextSize(0))); +} + +TYPED_TEST_P(CipherTest, Size_1) { + EXPECT_EQ(1, TypeParam::ciphertextSize(TypeParam::plaintextSize(1))); +} + +TYPED_TEST_P(CipherTest, Size_1024) { + EXPECT_EQ(1024, TypeParam::ciphertextSize(TypeParam::plaintextSize(1024))); + EXPECT_EQ(1024, TypeParam::plaintextSize(TypeParam::ciphertextSize(1024))); +} + +TYPED_TEST_P(CipherTest, Size_4096) { + EXPECT_EQ(4096, TypeParam::ciphertextSize(TypeParam::plaintextSize(4096))); + EXPECT_EQ(4096, TypeParam::plaintextSize(TypeParam::ciphertextSize(4096))); +} + +TYPED_TEST_P(CipherTest, Size_1048576) { + EXPECT_EQ(1048576, TypeParam::ciphertextSize(TypeParam::plaintextSize(1048576))); + EXPECT_EQ(1048576, TypeParam::plaintextSize(TypeParam::ciphertextSize(1048576))); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Empty) { + Data plaintext = this->CreateZeroes(0); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_1) { + Data plaintext = this->CreateZeroes(1); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_1) { + Data plaintext = this->CreateData(1); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_100) { + Data plaintext = this->CreateZeroes(100); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_100) { + Data plaintext = this->CreateData(100); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_1024) { + Data plaintext = this->CreateZeroes(1024); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_1024) { + Data plaintext = this->CreateData(1024); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_5000) { + Data plaintext = this->CreateZeroes(5000); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_5000) { + Data plaintext = this->CreateData(5000); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_1MB) { + Data plaintext = this->CreateZeroes(1048576); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_1MB) { + Data plaintext = this->CreateData(1048576); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_50MB) { + Data plaintext = this->CreateZeroes(52428800); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_50MB) { + Data plaintext = this->CreateData(52428800); + this->CheckEncryptThenDecryptIsIdentity(plaintext); +} + +REGISTER_TYPED_TEST_CASE_P(CipherTest, + Size_0, + Size_1, + Size_1024, + Size_4096, + Size_1048576, + EncryptThenDecrypt_Empty, + EncryptThenDecrypt_Zeroes_1, + EncryptThenDecrypt_Data_1, + EncryptThenDecrypt_Zeroes_100, + EncryptThenDecrypt_Data_100, + EncryptThenDecrypt_Zeroes_1024, + EncryptThenDecrypt_Data_1024, + EncryptThenDecrypt_Zeroes_5000, + EncryptThenDecrypt_Data_5000, + EncryptThenDecrypt_Zeroes_1MB, + EncryptThenDecrypt_Data_1MB, + EncryptThenDecrypt_Zeroes_50MB, + EncryptThenDecrypt_Data_50MB +); + +//TODO For authenticated ciphers, we need test cases checking that authentication fails on manipulations + +INSTANTIATE_TYPED_TEST_CASE_P(AES256_CFB, CipherTest, AES256_CFB); From 79283b868be529df5fa76eb9eda1e43a645a55eb Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 24 Apr 2015 19:13:56 +0200 Subject: [PATCH 04/41] Added test cases that a Cipher is indeterministic --- test/implementations/encrypted/CipherTest.cpp | 121 +++++++----------- 1 file changed, 47 insertions(+), 74 deletions(-) diff --git a/test/implementations/encrypted/CipherTest.cpp b/test/implementations/encrypted/CipherTest.cpp index 0f1429cf..27c21ad8 100644 --- a/test/implementations/encrypted/CipherTest.cpp +++ b/test/implementations/encrypted/CipherTest.cpp @@ -18,13 +18,30 @@ public: } void CheckEncryptThenDecryptIsIdentity(const Data &plaintext) { - Data ciphertext(Cipher::ciphertextSize(plaintext.size())); - Data decrypted(plaintext.size()); - Cipher::encrypt((byte*)plaintext.data(), plaintext.size(), (byte*)ciphertext.data(), this->encKey); - Cipher::decrypt((byte*)ciphertext.data(), (byte*) decrypted.data(), decrypted.size(), this->encKey); + Data ciphertext = Encrypt(plaintext); + Data decrypted = Decrypt(ciphertext); + EXPECT_EQ(plaintext.size(), decrypted.size()); EXPECT_EQ(0, std::memcmp(plaintext.data(), decrypted.data(), plaintext.size())); } + void CheckEncryptIsIndeterministic(const Data &plaintext) { + Data ciphertext = Encrypt(plaintext); + Data ciphertext2 = Encrypt(plaintext); + EXPECT_NE(0, std::memcmp(ciphertext.data(), ciphertext2.data(), ciphertext.size())); + } + + Data Encrypt(const Data &plaintext) { + Data ciphertext(Cipher::ciphertextSize(plaintext.size())); + Cipher::encrypt((byte*)plaintext.data(), plaintext.size(), (byte*)ciphertext.data(), this->encKey); + return ciphertext; + } + + Data Decrypt(const Data &ciphertext) { + Data decrypted(Cipher::plaintextSize(ciphertext.size())); + Cipher::decrypt((byte*)ciphertext.data(), (byte*) decrypted.data(), decrypted.size(), this->encKey); + return decrypted; + } + Data CreateZeroes(unsigned int size) { Data zeroes(size); zeroes.FillWithZeroes(); @@ -64,69 +81,34 @@ TYPED_TEST_P(CipherTest, Size_1048576) { EXPECT_EQ(1048576, TypeParam::plaintextSize(TypeParam::ciphertextSize(1048576))); } -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Empty) { - Data plaintext = this->CreateZeroes(0); - this->CheckEncryptThenDecryptIsIdentity(plaintext); +constexpr std::initializer_list SIZES = {0, 1, 100, 1024, 5000, 1048576, 52428800}; + +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes) { + for (auto size: SIZES) { + Data plaintext = this->CreateZeroes(size); + this->CheckEncryptThenDecryptIsIdentity(plaintext); + } } -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_1) { - Data plaintext = this->CreateZeroes(1); - this->CheckEncryptThenDecryptIsIdentity(plaintext); +TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data) { + for (auto size: SIZES) { + Data plaintext = this->CreateData(size); + this->CheckEncryptThenDecryptIsIdentity(plaintext); + } } -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_1) { - Data plaintext = this->CreateData(1); - this->CheckEncryptThenDecryptIsIdentity(plaintext); +TYPED_TEST_P(CipherTest, EncryptIsIndeterministic_Zeroes) { + for (auto size: SIZES) { + Data plaintext = this->CreateZeroes(size); + this->CheckEncryptIsIndeterministic(plaintext); + } } -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_100) { - Data plaintext = this->CreateZeroes(100); - this->CheckEncryptThenDecryptIsIdentity(plaintext); -} - -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_100) { - Data plaintext = this->CreateData(100); - this->CheckEncryptThenDecryptIsIdentity(plaintext); -} - -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_1024) { - Data plaintext = this->CreateZeroes(1024); - this->CheckEncryptThenDecryptIsIdentity(plaintext); -} - -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_1024) { - Data plaintext = this->CreateData(1024); - this->CheckEncryptThenDecryptIsIdentity(plaintext); -} - -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_5000) { - Data plaintext = this->CreateZeroes(5000); - this->CheckEncryptThenDecryptIsIdentity(plaintext); -} - -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_5000) { - Data plaintext = this->CreateData(5000); - this->CheckEncryptThenDecryptIsIdentity(plaintext); -} - -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_1MB) { - Data plaintext = this->CreateZeroes(1048576); - this->CheckEncryptThenDecryptIsIdentity(plaintext); -} - -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_1MB) { - Data plaintext = this->CreateData(1048576); - this->CheckEncryptThenDecryptIsIdentity(plaintext); -} - -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes_50MB) { - Data plaintext = this->CreateZeroes(52428800); - this->CheckEncryptThenDecryptIsIdentity(plaintext); -} - -TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Data_50MB) { - Data plaintext = this->CreateData(52428800); - this->CheckEncryptThenDecryptIsIdentity(plaintext); +TYPED_TEST_P(CipherTest, EncryptIsIndeterministic_Data) { + for (auto size: SIZES) { + Data plaintext = this->CreateData(size); + this->CheckEncryptIsIndeterministic(plaintext); + } } REGISTER_TYPED_TEST_CASE_P(CipherTest, @@ -135,19 +117,10 @@ REGISTER_TYPED_TEST_CASE_P(CipherTest, Size_1024, Size_4096, Size_1048576, - EncryptThenDecrypt_Empty, - EncryptThenDecrypt_Zeroes_1, - EncryptThenDecrypt_Data_1, - EncryptThenDecrypt_Zeroes_100, - EncryptThenDecrypt_Data_100, - EncryptThenDecrypt_Zeroes_1024, - EncryptThenDecrypt_Data_1024, - EncryptThenDecrypt_Zeroes_5000, - EncryptThenDecrypt_Data_5000, - EncryptThenDecrypt_Zeroes_1MB, - EncryptThenDecrypt_Data_1MB, - EncryptThenDecrypt_Zeroes_50MB, - EncryptThenDecrypt_Data_50MB + EncryptThenDecrypt_Zeroes, + EncryptThenDecrypt_Data, + EncryptIsIndeterministic_Zeroes, + EncryptIsIndeterministic_Data ); //TODO For authenticated ciphers, we need test cases checking that authentication fails on manipulations From 5adcf4aca1e555688f6105a9633f6dff41ef53ee Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 24 Apr 2015 21:08:36 +0200 Subject: [PATCH 05/41] Allow decryption interface to fail (needed for authenticated ciphers later) --- implementations/encrypted/EncryptedBlock.h | 26 ++++++++----------- .../encrypted/EncryptedBlockStore.h | 8 +++++- .../encrypted/ciphers/AES256_CFB.cpp | 20 ++++++++------ .../encrypted/ciphers/AES256_CFB.h | 6 +++-- test/implementations/encrypted/CipherTest.cpp | 23 +++++++++++----- 5 files changed, 50 insertions(+), 33 deletions(-) diff --git a/implementations/encrypted/EncryptedBlock.h b/implementations/encrypted/EncryptedBlock.h index ee64a56b..b5414217 100644 --- a/implementations/encrypted/EncryptedBlock.h +++ b/implementations/encrypted/EncryptedBlock.h @@ -8,6 +8,7 @@ #include "messmer/cpp-utils/macros.h" #include +#include namespace blockstore { namespace encrypted { @@ -19,9 +20,9 @@ template class EncryptedBlock: public Block { public: static std::unique_ptr TryCreateNew(BlockStore *baseBlockStore, const Key &key, Data data, const typename Cipher::EncryptionKey &encKey); + static std::unique_ptr TryDecrypt(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key); //TODO Storing key twice (in parent class and in object pointed to). Once would be enough. - EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key); EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key, Data plaintextData); virtual ~EncryptedBlock(); @@ -40,7 +41,6 @@ private: bool _dataChanged; void _encryptToBaseBlock(); - void _decryptFromBaseBlock(); DISALLOW_COPY_AND_ASSIGN(EncryptedBlock); }; @@ -49,8 +49,7 @@ private: template std::unique_ptr> EncryptedBlock::TryCreateNew(BlockStore *baseBlockStore, const Key &key, Data data, const typename Cipher::EncryptionKey &encKey) { - Data encrypted(Cipher::ciphertextSize(data.size())); - Cipher::encrypt((byte*)data.data(), data.size(), (byte*)encrypted.data(), encKey); + Data encrypted = Cipher::encrypt((byte*)data.data(), data.size(), encKey); auto baseBlock = baseBlockStore->tryCreate(key, std::move(encrypted)); if (baseBlock.get() == nullptr) { //TODO Test this code branch @@ -61,9 +60,13 @@ std::unique_ptr> EncryptedBlock::TryCreateNew(Blo } template -EncryptedBlock::EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &encKey) - :EncryptedBlock(std::move(baseBlock), encKey, Data(Cipher::plaintextSize(baseBlock->size()))) { - _decryptFromBaseBlock(); +std::unique_ptr> EncryptedBlock::TryDecrypt(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &encKey) { + //TODO Change BlockStore so we can read their "class Data" objects instead of "void *data()", and then we can change the Cipher interface to take Data objects instead of "byte *" + size + boost::optional plaintext = Cipher::decrypt((byte*)baseBlock->data(), baseBlock->size(), encKey); + if(!plaintext) { + return nullptr; + } + return std::make_unique>(std::move(baseBlock), encKey, std::move(*plaintext)); } template @@ -103,17 +106,10 @@ size_t EncryptedBlock::size() const { return _plaintextData.size(); } -template -void EncryptedBlock::_decryptFromBaseBlock() { - //TODO Change BlockStore so we can read their "class Data" objects instead of "void *data()", and then we can change the Cipher interface to take Data objects instead of "byte *" + size - Cipher::decrypt((byte*)_baseBlock->data(), (byte*)_plaintextData.data(), _plaintextData.size(), _encKey); -} - template void EncryptedBlock::_encryptToBaseBlock() { if (_dataChanged) { - Data encrypted(Cipher::ciphertextSize(_plaintextData.size())); - Cipher::encrypt((byte*)_plaintextData.data(), _plaintextData.size(), (byte*)encrypted.data(), _encKey); + Data encrypted = Cipher::encrypt((byte*)_plaintextData.data(), _plaintextData.size(), _encKey); _baseBlock->write(encrypted.data(), 0, encrypted.size()); _dataChanged = false; } diff --git a/implementations/encrypted/EncryptedBlockStore.h b/implementations/encrypted/EncryptedBlockStore.h index a8ce51ac..c116200d 100644 --- a/implementations/encrypted/EncryptedBlockStore.h +++ b/implementations/encrypted/EncryptedBlockStore.h @@ -6,6 +6,7 @@ #include #include #include "EncryptedBlock.h" +#include namespace blockstore { namespace encrypted { @@ -51,7 +52,12 @@ std::unique_ptr EncryptedBlockStore::load(const Key &key) { if (block.get() == nullptr) { return nullptr; } - return std::make_unique>(std::move(block), _encKey); + auto encBlock = EncryptedBlock::TryDecrypt(std::move(block), _encKey); + if (encBlock.get() == nullptr) { + //TODO Think about how to do logging + + } + return std::move(encBlock); } template diff --git a/implementations/encrypted/ciphers/AES256_CFB.cpp b/implementations/encrypted/ciphers/AES256_CFB.cpp index bad5562f..81a3ea41 100644 --- a/implementations/encrypted/ciphers/AES256_CFB.cpp +++ b/implementations/encrypted/ciphers/AES256_CFB.cpp @@ -9,18 +9,22 @@ namespace encrypted { constexpr unsigned int AES256_CFB::IV_SIZE; -void AES256_CFB::encrypt(const byte *plaintext, unsigned int plaintextSize, byte *ciphertext, const EncryptionKey &encKey) { +Data AES256_CFB::encrypt(const byte *plaintext, unsigned int plaintextSize, const EncryptionKey &encKey) { FixedSizeData iv = FixedSizeData::CreateRandom(); auto encryption = CFB_Mode::Encryption(encKey.data(), encKey.BINARY_LENGTH, iv.data()); - std::memcpy(ciphertext, iv.data(), IV_SIZE); - encryption.ProcessData(ciphertext + IV_SIZE, plaintext, plaintextSize); + Data ciphertext(ciphertextSize(plaintextSize)); + std::memcpy(ciphertext.data(), iv.data(), IV_SIZE); + encryption.ProcessData((byte*)ciphertext.data() + IV_SIZE, plaintext, plaintextSize); + return ciphertext; } -void AES256_CFB::decrypt(const byte *ciphertext, byte *plaintext, unsigned int plaintextSize, const EncryptionKey &encKey) { - const byte *iv = ciphertext; - const byte *data = ciphertext + IV_SIZE; - auto decryption = CFB_Mode::Decryption((byte*)encKey.data(), encKey.BINARY_LENGTH, iv); - decryption.ProcessData(plaintext, data, plaintextSize); +boost::optional AES256_CFB::decrypt(const byte *ciphertext, unsigned int ciphertextSize, const EncryptionKey &encKey) { + const byte *ciphertextIV = ciphertext; + const byte *ciphertextData = ciphertext + IV_SIZE; + auto decryption = CFB_Mode::Decryption((byte*)encKey.data(), encKey.BINARY_LENGTH, ciphertextIV); + Data plaintext(plaintextSize(ciphertextSize)); + decryption.ProcessData((byte*)plaintext.data(), ciphertextData, plaintext.size()); + return std::move(plaintext); } } diff --git a/implementations/encrypted/ciphers/AES256_CFB.h b/implementations/encrypted/ciphers/AES256_CFB.h index 46d4aece..5a817f35 100644 --- a/implementations/encrypted/ciphers/AES256_CFB.h +++ b/implementations/encrypted/ciphers/AES256_CFB.h @@ -3,7 +3,9 @@ #define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHERS_AES256_CFB_H_ #include "../../../utils/FixedSizeData.h" +#include "../../../utils/Data.h" #include +#include namespace blockstore { namespace encrypted { @@ -24,8 +26,8 @@ public: return ciphertextBlockSize - IV_SIZE; } - static void encrypt(const byte *plaintext, unsigned int plaintextSize, byte *ciphertext, const EncryptionKey &key); - static void decrypt(const byte *ciphertext, byte *plaintext, unsigned int plaintextSize, const EncryptionKey &key); + static Data encrypt(const byte *plaintext, unsigned int plaintextSize, const EncryptionKey &encKey); + static boost::optional decrypt(const byte *ciphertext, unsigned int ciphertextSize, const EncryptionKey &encKey); private: static constexpr unsigned int IV_SIZE = CryptoPP::AES::BLOCKSIZE; diff --git a/test/implementations/encrypted/CipherTest.cpp b/test/implementations/encrypted/CipherTest.cpp index 27c21ad8..f1bccee3 100644 --- a/test/implementations/encrypted/CipherTest.cpp +++ b/test/implementations/encrypted/CipherTest.cpp @@ -30,16 +30,17 @@ public: EXPECT_NE(0, std::memcmp(ciphertext.data(), ciphertext2.data(), ciphertext.size())); } + void CheckEncryptedSize(const Data &plaintext) { + Data ciphertext = Encrypt(plaintext); + EXPECT_EQ(Cipher::ciphertextSize(plaintext.size()), ciphertext.size()); + } + Data Encrypt(const Data &plaintext) { - Data ciphertext(Cipher::ciphertextSize(plaintext.size())); - Cipher::encrypt((byte*)plaintext.data(), plaintext.size(), (byte*)ciphertext.data(), this->encKey); - return ciphertext; + return Cipher::encrypt((byte*)plaintext.data(), plaintext.size(), this->encKey); } Data Decrypt(const Data &ciphertext) { - Data decrypted(Cipher::plaintextSize(ciphertext.size())); - Cipher::decrypt((byte*)ciphertext.data(), (byte*) decrypted.data(), decrypted.size(), this->encKey); - return decrypted; + return Cipher::decrypt((byte*)ciphertext.data(), ciphertext.size(), this->encKey).value(); } Data CreateZeroes(unsigned int size) { @@ -111,6 +112,13 @@ TYPED_TEST_P(CipherTest, EncryptIsIndeterministic_Data) { } } +TYPED_TEST_P(CipherTest, EncryptedSize) { + for (auto size: SIZES) { + Data plaintext = this->CreateData(size); + this->CheckEncryptedSize(plaintext); + } +} + REGISTER_TYPED_TEST_CASE_P(CipherTest, Size_0, Size_1, @@ -120,7 +128,8 @@ REGISTER_TYPED_TEST_CASE_P(CipherTest, EncryptThenDecrypt_Zeroes, EncryptThenDecrypt_Data, EncryptIsIndeterministic_Zeroes, - EncryptIsIndeterministic_Data + EncryptIsIndeterministic_Data, + EncryptedSize ); //TODO For authenticated ciphers, we need test cases checking that authentication fails on manipulations From 008c6f7ab7d0d16150f002e2c10349bd0888d40c Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 24 Apr 2015 21:30:54 +0200 Subject: [PATCH 06/41] Add Cipher concept --- implementations/encrypted/EncryptedBlock.h | 2 ++ .../encrypted/ciphers/AES256_CFB.h | 5 +-- implementations/encrypted/ciphers/Cipher.h | 32 +++++++++++++++++++ test/implementations/encrypted/CipherTest.cpp | 4 ++- 4 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 implementations/encrypted/ciphers/Cipher.h diff --git a/implementations/encrypted/EncryptedBlock.h b/implementations/encrypted/EncryptedBlock.h index b5414217..cb0e10a4 100644 --- a/implementations/encrypted/EncryptedBlock.h +++ b/implementations/encrypted/EncryptedBlock.h @@ -9,6 +9,7 @@ #include "messmer/cpp-utils/macros.h" #include #include +#include "ciphers/Cipher.h" namespace blockstore { namespace encrypted { @@ -19,6 +20,7 @@ template class EncryptedBlockStore; template class EncryptedBlock: public Block { public: + BOOST_CONCEPT_ASSERT((CipherConcept)); static std::unique_ptr TryCreateNew(BlockStore *baseBlockStore, const Key &key, Data data, const typename Cipher::EncryptionKey &encKey); static std::unique_ptr TryDecrypt(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key); diff --git a/implementations/encrypted/ciphers/AES256_CFB.h b/implementations/encrypted/ciphers/AES256_CFB.h index 5a817f35..154106ce 100644 --- a/implementations/encrypted/ciphers/AES256_CFB.h +++ b/implementations/encrypted/ciphers/AES256_CFB.h @@ -6,6 +6,7 @@ #include "../../../utils/Data.h" #include #include +#include "Cipher.h" namespace blockstore { namespace encrypted { @@ -13,11 +14,11 @@ namespace encrypted { //TODO Add contract/interface for ciphers class AES256_CFB { public: + BOOST_CONCEPT_ASSERT((CipherConcept)); + using EncryptionKey = FixedSizeData<32>; static_assert(32 == CryptoPP::AES::MAX_KEYLENGTH, "If AES offered larger keys, we should offer a variant with it"); - AES256_CFB(const EncryptionKey &key); - static constexpr unsigned int ciphertextSize(unsigned int plaintextBlockSize) { return plaintextBlockSize + IV_SIZE; } diff --git a/implementations/encrypted/ciphers/Cipher.h b/implementations/encrypted/ciphers/Cipher.h new file mode 100644 index 00000000..08936427 --- /dev/null +++ b/implementations/encrypted/ciphers/Cipher.h @@ -0,0 +1,32 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHERS_CIPHER_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHERS_CIPHER_H_ + +#include +#include + +namespace blockstore { +namespace encrypted { + +template +struct CipherConcept { +public: + BOOST_CONCEPT_USAGE(CipherConcept) { + same_type(UINT32_C(0), X::ciphertextSize(UINT32_C(5))); + same_type(UINT32_C(0), X::plaintextSize(UINT32_C(5))); + typename X::EncryptionKey key = X::EncryptionKey::CreateRandom(); + same_type(Data(0), X::encrypt((byte*)nullptr, UINT32_C(0), key)); + same_type(boost::optional(Data(0)), X::decrypt((byte*)nullptr, UINT32_C(0), key)); + } + +private: + // Type deduction will fail unless the arguments have the same type. + template void same_type(T const&, T const&); +}; + +} +} + + + +#endif diff --git a/test/implementations/encrypted/CipherTest.cpp b/test/implementations/encrypted/CipherTest.cpp index f1bccee3..a4385998 100644 --- a/test/implementations/encrypted/CipherTest.cpp +++ b/test/implementations/encrypted/CipherTest.cpp @@ -1,5 +1,6 @@ #include #include "../../../implementations/encrypted/ciphers/AES256_CFB.h" +#include "../../../implementations/encrypted/ciphers/Cipher.h" #include "../../testutils/DataBlockFixture.h" #include "../../../utils/Data.h" @@ -10,6 +11,7 @@ using blockstore::Data; template class CipherTest: public ::testing::Test { public: + BOOST_CONCEPT_ASSERT((CipherConcept)); typename Cipher::EncryptionKey encKey = createRandomKey(); static typename Cipher::EncryptionKey createRandomKey(int seed = 0) { @@ -82,7 +84,7 @@ TYPED_TEST_P(CipherTest, Size_1048576) { EXPECT_EQ(1048576, TypeParam::plaintextSize(TypeParam::ciphertextSize(1048576))); } -constexpr std::initializer_list SIZES = {0, 1, 100, 1024, 5000, 1048576, 52428800}; +constexpr std::initializer_list SIZES = {0, 1, 100, 1024, 5000, 1048576, 20971520}; TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes) { for (auto size: SIZES) { From 19e5f03de3c9556299b90ddc9eddd56d6370f725 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 24 Apr 2015 21:33:25 +0200 Subject: [PATCH 07/41] Simplified CipherTest --- test/implementations/encrypted/CipherTest.cpp | 36 +++++-------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/test/implementations/encrypted/CipherTest.cpp b/test/implementations/encrypted/CipherTest.cpp index a4385998..f7471d60 100644 --- a/test/implementations/encrypted/CipherTest.cpp +++ b/test/implementations/encrypted/CipherTest.cpp @@ -61,31 +61,15 @@ public: TYPED_TEST_CASE_P(CipherTest); -TYPED_TEST_P(CipherTest, Size_0) { - EXPECT_EQ(0, TypeParam::ciphertextSize(TypeParam::plaintextSize(0))); -} - -TYPED_TEST_P(CipherTest, Size_1) { - EXPECT_EQ(1, TypeParam::ciphertextSize(TypeParam::plaintextSize(1))); -} - -TYPED_TEST_P(CipherTest, Size_1024) { - EXPECT_EQ(1024, TypeParam::ciphertextSize(TypeParam::plaintextSize(1024))); - EXPECT_EQ(1024, TypeParam::plaintextSize(TypeParam::ciphertextSize(1024))); -} - -TYPED_TEST_P(CipherTest, Size_4096) { - EXPECT_EQ(4096, TypeParam::ciphertextSize(TypeParam::plaintextSize(4096))); - EXPECT_EQ(4096, TypeParam::plaintextSize(TypeParam::ciphertextSize(4096))); -} - -TYPED_TEST_P(CipherTest, Size_1048576) { - EXPECT_EQ(1048576, TypeParam::ciphertextSize(TypeParam::plaintextSize(1048576))); - EXPECT_EQ(1048576, TypeParam::plaintextSize(TypeParam::ciphertextSize(1048576))); -} - constexpr std::initializer_list SIZES = {0, 1, 100, 1024, 5000, 1048576, 20971520}; +TYPED_TEST_P(CipherTest, Size) { + for (auto size: SIZES) { + EXPECT_EQ(size, TypeParam::ciphertextSize(TypeParam::plaintextSize(size))); + EXPECT_EQ(size, TypeParam::plaintextSize(TypeParam::ciphertextSize(size))); + } +} + TYPED_TEST_P(CipherTest, EncryptThenDecrypt_Zeroes) { for (auto size: SIZES) { Data plaintext = this->CreateZeroes(size); @@ -122,11 +106,7 @@ TYPED_TEST_P(CipherTest, EncryptedSize) { } REGISTER_TYPED_TEST_CASE_P(CipherTest, - Size_0, - Size_1, - Size_1024, - Size_4096, - Size_1048576, + Size, EncryptThenDecrypt_Zeroes, EncryptThenDecrypt_Data, EncryptIsIndeterministic_Zeroes, From 6e28af31054178a3ae9584a91d381e03e75c7a42 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 24 Apr 2015 21:33:45 +0200 Subject: [PATCH 08/41] TODOs --- implementations/encrypted/ciphers/AES256_CFB.h | 1 - 1 file changed, 1 deletion(-) diff --git a/implementations/encrypted/ciphers/AES256_CFB.h b/implementations/encrypted/ciphers/AES256_CFB.h index 154106ce..39f36129 100644 --- a/implementations/encrypted/ciphers/AES256_CFB.h +++ b/implementations/encrypted/ciphers/AES256_CFB.h @@ -11,7 +11,6 @@ namespace blockstore { namespace encrypted { -//TODO Add contract/interface for ciphers class AES256_CFB { public: BOOST_CONCEPT_ASSERT((CipherConcept)); From 0335b243fb28b3475f91b118090b808050dc9087 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 24 Apr 2015 23:02:14 +0200 Subject: [PATCH 09/41] Implemented AES256_GCM --- .../encrypted/ciphers/AES256_GCM.cpp | 49 +++++++++++++++++++ .../encrypted/ciphers/AES256_GCM.h | 40 +++++++++++++++ test/implementations/encrypted/CipherTest.cpp | 2 + 3 files changed, 91 insertions(+) create mode 100644 implementations/encrypted/ciphers/AES256_GCM.cpp create mode 100644 implementations/encrypted/ciphers/AES256_GCM.h diff --git a/implementations/encrypted/ciphers/AES256_GCM.cpp b/implementations/encrypted/ciphers/AES256_GCM.cpp new file mode 100644 index 00000000..fd7b5788 --- /dev/null +++ b/implementations/encrypted/ciphers/AES256_GCM.cpp @@ -0,0 +1,49 @@ +#include +#include "AES256_GCM.h" + +using CryptoPP::GCM; +using CryptoPP::AES; +using CryptoPP::AuthenticatedEncryptionFilter; +using CryptoPP::AuthenticatedDecryptionFilter; +using CryptoPP::ArraySource; +using CryptoPP::ArraySink; +using CryptoPP::GCM_64K_Tables; + +namespace blockstore { +namespace encrypted { + +constexpr unsigned int AES256_GCM::IV_SIZE; + +Data AES256_GCM::encrypt(const byte *plaintext, unsigned int plaintextSize, const EncryptionKey &encKey) { + FixedSizeData iv = FixedSizeData::CreateRandom(); + GCM::Encryption encryption; + encryption.SetKeyWithIV(encKey.data(), encKey.BINARY_LENGTH, iv.data(), IV_SIZE); + Data ciphertext(ciphertextSize(plaintextSize)); + + std::memcpy(ciphertext.data(), iv.data(), IV_SIZE); + ArraySource(plaintext, plaintextSize, true, + new AuthenticatedEncryptionFilter(encryption, + new ArraySink((byte*)ciphertext.data() + IV_SIZE, ciphertext.size() - IV_SIZE), + false, TAG_SIZE + ) + ); + return ciphertext; +} + +boost::optional AES256_GCM::decrypt(const byte *ciphertext, unsigned int ciphertextSize, const EncryptionKey &encKey) { + const byte *ciphertextIV = ciphertext; + const byte *ciphertextData = ciphertext + IV_SIZE; + GCM::Decryption decryption; + decryption.SetKeyWithIV((byte*)encKey.data(), encKey.BINARY_LENGTH, ciphertextIV, IV_SIZE); + Data plaintext(plaintextSize(ciphertextSize)); + + ArraySource((byte*)ciphertextData, ciphertextSize - IV_SIZE, true, + new AuthenticatedDecryptionFilter(decryption, + new ArraySink((byte*)plaintext.data(), plaintext.size()) + ) + ); + return std::move(plaintext); +} + +} +} diff --git a/implementations/encrypted/ciphers/AES256_GCM.h b/implementations/encrypted/ciphers/AES256_GCM.h new file mode 100644 index 00000000..f13c2a03 --- /dev/null +++ b/implementations/encrypted/ciphers/AES256_GCM.h @@ -0,0 +1,40 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHERS_AES256_GCM_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHERS_AES256_GCM_H_ + +#include "../../../utils/FixedSizeData.h" +#include "../../../utils/Data.h" +#include +#include +#include "Cipher.h" + +namespace blockstore { +namespace encrypted { + +class AES256_GCM { +public: + BOOST_CONCEPT_ASSERT((CipherConcept)); + + using EncryptionKey = FixedSizeData<32>; + static_assert(32 == CryptoPP::AES::MAX_KEYLENGTH, "If AES offered larger keys, we should offer a variant with it"); + + static constexpr unsigned int ciphertextSize(unsigned int plaintextBlockSize) { + return plaintextBlockSize + IV_SIZE + TAG_SIZE; + } + + static constexpr unsigned int plaintextSize(unsigned int ciphertextBlockSize) { + return ciphertextBlockSize - IV_SIZE - TAG_SIZE; + } + + static Data encrypt(const byte *plaintext, unsigned int plaintextSize, const EncryptionKey &encKey); + static boost::optional decrypt(const byte *ciphertext, unsigned int ciphertextSize, const EncryptionKey &encKey); + +private: + static constexpr unsigned int IV_SIZE = CryptoPP::AES::BLOCKSIZE; + static constexpr unsigned int TAG_SIZE = 16; +}; + +} +} + +#endif diff --git a/test/implementations/encrypted/CipherTest.cpp b/test/implementations/encrypted/CipherTest.cpp index f7471d60..e650fcb4 100644 --- a/test/implementations/encrypted/CipherTest.cpp +++ b/test/implementations/encrypted/CipherTest.cpp @@ -1,5 +1,6 @@ #include #include "../../../implementations/encrypted/ciphers/AES256_CFB.h" +#include "../../../implementations/encrypted/ciphers/AES256_GCM.h" #include "../../../implementations/encrypted/ciphers/Cipher.h" #include "../../testutils/DataBlockFixture.h" @@ -117,3 +118,4 @@ REGISTER_TYPED_TEST_CASE_P(CipherTest, //TODO For authenticated ciphers, we need test cases checking that authentication fails on manipulations INSTANTIATE_TYPED_TEST_CASE_P(AES256_CFB, CipherTest, AES256_CFB); +INSTANTIATE_TYPED_TEST_CASE_P(AES256_GCM, CipherTest, AES256_GCM); From e056a65b48c3e90c05a405c84fc31c8133c7b464 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 24 Apr 2015 23:55:52 +0200 Subject: [PATCH 10/41] - Added test cases for authenticated ciphers - Fixed corner case for AES256_GCM when decrypt is called on data that can't hold IV and TAG --- .../encrypted/ciphers/AES256_GCM.cpp | 21 +++- test/implementations/encrypted/CipherTest.cpp | 113 +++++++++++++++++- 2 files changed, 125 insertions(+), 9 deletions(-) diff --git a/implementations/encrypted/ciphers/AES256_GCM.cpp b/implementations/encrypted/ciphers/AES256_GCM.cpp index fd7b5788..6118977c 100644 --- a/implementations/encrypted/ciphers/AES256_GCM.cpp +++ b/implementations/encrypted/ciphers/AES256_GCM.cpp @@ -8,6 +8,7 @@ using CryptoPP::AuthenticatedDecryptionFilter; using CryptoPP::ArraySource; using CryptoPP::ArraySink; using CryptoPP::GCM_64K_Tables; +using CryptoPP::HashVerificationFilter; namespace blockstore { namespace encrypted { @@ -31,18 +32,26 @@ Data AES256_GCM::encrypt(const byte *plaintext, unsigned int plaintextSize, cons } boost::optional AES256_GCM::decrypt(const byte *ciphertext, unsigned int ciphertextSize, const EncryptionKey &encKey) { + if (ciphertextSize < IV_SIZE + TAG_SIZE) { + return boost::none; + } + const byte *ciphertextIV = ciphertext; const byte *ciphertextData = ciphertext + IV_SIZE; GCM::Decryption decryption; decryption.SetKeyWithIV((byte*)encKey.data(), encKey.BINARY_LENGTH, ciphertextIV, IV_SIZE); Data plaintext(plaintextSize(ciphertextSize)); - ArraySource((byte*)ciphertextData, ciphertextSize - IV_SIZE, true, - new AuthenticatedDecryptionFilter(decryption, - new ArraySink((byte*)plaintext.data(), plaintext.size()) - ) - ); - return std::move(plaintext); + try { + ArraySource((byte*)ciphertextData, ciphertextSize - IV_SIZE, true, + new AuthenticatedDecryptionFilter(decryption, + new ArraySink((byte*)plaintext.data(), plaintext.size()) + ) + ); + return std::move(plaintext); + } catch (const HashVerificationFilter::HashVerificationFailed &e) { + return boost::none; + } } } diff --git a/test/implementations/encrypted/CipherTest.cpp b/test/implementations/encrypted/CipherTest.cpp index e650fcb4..a505c7b3 100644 --- a/test/implementations/encrypted/CipherTest.cpp +++ b/test/implementations/encrypted/CipherTest.cpp @@ -6,6 +6,8 @@ #include "../../testutils/DataBlockFixture.h" #include "../../../utils/Data.h" + #include + using namespace blockstore::encrypted; using blockstore::Data; @@ -46,13 +48,13 @@ public: return Cipher::decrypt((byte*)ciphertext.data(), ciphertext.size(), this->encKey).value(); } - Data CreateZeroes(unsigned int size) { + static Data CreateZeroes(unsigned int size) { Data zeroes(size); zeroes.FillWithZeroes(); return zeroes; } - Data CreateData(unsigned int size, unsigned int seed = 0) { + static Data CreateData(unsigned int size, unsigned int seed = 0) { DataBlockFixture data(size, seed); Data result(size); std::memcpy(result.data(), data.data(), size); @@ -115,7 +117,112 @@ REGISTER_TYPED_TEST_CASE_P(CipherTest, EncryptedSize ); -//TODO For authenticated ciphers, we need test cases checking that authentication fails on manipulations +template +class AuthenticatedCipherTest: public CipherTest { +public: + void ExpectDoesntDecrypt(const Data &ciphertext) { + auto decrypted = Cipher::decrypt((byte*)ciphertext.data(), ciphertext.size(), this->encKey); + EXPECT_FALSE(decrypted); + } + + Data zeroes1 = CipherTest::CreateZeroes(1); + Data plaintext1 = CipherTest::CreateData(1); + Data zeroes2 = CipherTest::CreateZeroes(100 * 1024); + Data plaintext2 = CipherTest::CreateData(100 * 1024); +}; + +TYPED_TEST_CASE_P(AuthenticatedCipherTest); + +TYPED_TEST_P(AuthenticatedCipherTest, ModifyFirstByte_Zeroes_Size1) { + Data ciphertext = this->Encrypt(this->zeroes1); + *(byte*)ciphertext.data() = *(byte*)ciphertext.data() + 1; + this->ExpectDoesntDecrypt(ciphertext); +} + +TYPED_TEST_P(AuthenticatedCipherTest, ModifyFirstByte_Data_Size1) { + Data ciphertext = this->Encrypt(this->plaintext1); + *(byte*)ciphertext.data() = *(byte*)ciphertext.data() + 1; + this->ExpectDoesntDecrypt(ciphertext); +} + +TYPED_TEST_P(AuthenticatedCipherTest, ModifyFirstByte_Zeroes) { + Data ciphertext = this->Encrypt(this->zeroes2); + *(byte*)ciphertext.data() = *(byte*)ciphertext.data() + 1; + this->ExpectDoesntDecrypt(ciphertext); +} + +TYPED_TEST_P(AuthenticatedCipherTest, ModifyFirstByte_Data) { + Data ciphertext = this->Encrypt(this->plaintext2); + *(byte*)ciphertext.data() = *(byte*)ciphertext.data() + 1; + this->ExpectDoesntDecrypt(ciphertext); +} + +TYPED_TEST_P(AuthenticatedCipherTest, ModifyLastByte_Zeroes) { + Data ciphertext = this->Encrypt(this->zeroes2); + ((byte*)ciphertext.data())[ciphertext.size() - 1] = ((byte*)ciphertext.data())[ciphertext.size() - 1] + 1; + this->ExpectDoesntDecrypt(ciphertext); +} + +TYPED_TEST_P(AuthenticatedCipherTest, ModifyLastByte_Data) { + Data ciphertext = this->Encrypt(this->plaintext2); + ((byte*)ciphertext.data())[ciphertext.size() - 1] = ((byte*)ciphertext.data())[ciphertext.size() - 1] + 1; + this->ExpectDoesntDecrypt(ciphertext); +} + +TYPED_TEST_P(AuthenticatedCipherTest, ModifyMiddleByte_Zeroes) { + Data ciphertext = this->Encrypt(this->zeroes2); + ((byte*)ciphertext.data())[ciphertext.size()/2] = ((byte*)ciphertext.data())[ciphertext.size()/2] + 1; + this->ExpectDoesntDecrypt(ciphertext); +} + +TYPED_TEST_P(AuthenticatedCipherTest, ModifyMiddleByte_Data) { + Data ciphertext = this->Encrypt(this->plaintext2); + ((byte*)ciphertext.data())[ciphertext.size()/2] = ((byte*)ciphertext.data())[ciphertext.size()/2] + 1; + this->ExpectDoesntDecrypt(ciphertext); +} + +TYPED_TEST_P(AuthenticatedCipherTest, TryDecryptZeroesData) { + this->ExpectDoesntDecrypt(this->zeroes2); +} + +TYPED_TEST_P(AuthenticatedCipherTest, TryDecryptRandomData) { + this->ExpectDoesntDecrypt(this->plaintext2); +} + +TYPED_TEST_P(AuthenticatedCipherTest, TryDecryptDataThatIsTooSmall) { + Data tooSmallCiphertext(TypeParam::ciphertextSize(0) - 1); + this->ExpectDoesntDecrypt(tooSmallCiphertext); +} + +TYPED_TEST_P(AuthenticatedCipherTest, TryDecryptDataThatIsMuchTooSmall_0) { + static_assert(TypeParam::ciphertextSize(0) > 0, "If this fails, the test case doesn't make sense."); + Data tooSmallCiphertext(0); + this->ExpectDoesntDecrypt(tooSmallCiphertext); +} + +TYPED_TEST_P(AuthenticatedCipherTest, TryDecryptDataThatIsMuchTooSmall_1) { + static_assert(TypeParam::ciphertextSize(0) > 1, "If this fails, the test case doesn't make sense."); + Data tooSmallCiphertext(1); + this->ExpectDoesntDecrypt(tooSmallCiphertext); +} + +REGISTER_TYPED_TEST_CASE_P(AuthenticatedCipherTest, + ModifyFirstByte_Zeroes_Size1, + ModifyFirstByte_Zeroes, + ModifyFirstByte_Data_Size1, + ModifyFirstByte_Data, + ModifyLastByte_Zeroes, + ModifyLastByte_Data, + ModifyMiddleByte_Zeroes, + ModifyMiddleByte_Data, + TryDecryptZeroesData, + TryDecryptRandomData, + TryDecryptDataThatIsTooSmall, + TryDecryptDataThatIsMuchTooSmall_0, + TryDecryptDataThatIsMuchTooSmall_1 +); + INSTANTIATE_TYPED_TEST_CASE_P(AES256_CFB, CipherTest, AES256_CFB); INSTANTIATE_TYPED_TEST_CASE_P(AES256_GCM, CipherTest, AES256_GCM); +INSTANTIATE_TYPED_TEST_CASE_P(AES256_GCM, AuthenticatedCipherTest, AES256_GCM); From 60442a46f0186c83e85658a759b4e663bc2aabc2 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sat, 25 Apr 2015 00:42:10 +0200 Subject: [PATCH 11/41] Upgrade dependencies --- biicode.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/biicode.conf b/biicode.conf index 89337881..29a2833a 100644 --- a/biicode.conf +++ b/biicode.conf @@ -1,9 +1,9 @@ # Biicode configuration file [requirements] - cryptopp/cryptopp: 8 + cryptopp/cryptopp: 9 google/gmock: 2 - google/gtest: 10 + google/gtest: 11 messmer/cmake: 3 messmer/cpp-utils: 2 messmer/parallelaccessstore: 0 From 59bfddc18f18929b0304178158940dcc18473b6e Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sat, 25 Apr 2015 02:48:41 +0200 Subject: [PATCH 12/41] Moved Data, FixedSizeData and DataBlockFixture to cpp-utils --- implementations/caching/CachingBlockStore.cpp | 1 + implementations/caching/CachingBlockStore.h | 4 +- implementations/caching/NewBlock.cpp | 1 + implementations/caching/NewBlock.h | 6 +- implementations/encrypted/EncryptedBlock.h | 18 +- .../encrypted/EncryptedBlockStore.h | 4 +- .../encrypted/ciphers/AES256_CFB.cpp | 2 + .../encrypted/ciphers/AES256_CFB.h | 10 +- .../encrypted/ciphers/AES256_GCM.cpp | 2 + .../encrypted/ciphers/AES256_GCM.h | 11 +- implementations/encrypted/ciphers/Cipher.h | 4 +- implementations/inmemory/InMemoryBlock.cpp | 1 + implementations/inmemory/InMemoryBlock.h | 6 +- .../inmemory/InMemoryBlockStore.cpp | 1 + implementations/inmemory/InMemoryBlockStore.h | 2 +- implementations/ondisk/OnDiskBlock.cpp | 9 +- implementations/ondisk/OnDiskBlock.h | 8 +- implementations/ondisk/OnDiskBlockStore.cpp | 1 + implementations/ondisk/OnDiskBlockStore.h | 2 +- .../ParallelAccessBlockStore.cpp | 2 +- .../parallelaccess/ParallelAccessBlockStore.h | 2 +- implementations/testfake/FakeBlock.cpp | 1 + implementations/testfake/FakeBlock.h | 6 +- implementations/testfake/FakeBlockStore.cpp | 1 + implementations/testfake/FakeBlockStore.h | 12 +- interface/BlockStore.h | 6 +- test/implementations/encrypted/CipherTest.cpp | 7 +- .../OnDiskBlockTest/OnDiskBlockCreateTest.cpp | 14 +- .../OnDiskBlockTest/OnDiskBlockFlushTest.cpp | 15 +- .../OnDiskBlockTest/OnDiskBlockLoadTest.cpp | 15 +- .../helpers/BlockStoreWithRandomKeysTest.cpp | 4 +- test/testutils/BlockStoreTest.h | 34 ++-- test/testutils/BlockStoreTest_Data.h | 30 ++-- test/testutils/BlockStoreTest_Size.h | 45 ++--- test/testutils/BlockStoreWithRandomKeysTest.h | 21 ++- test/testutils/DataBlockFixture.cpp | 87 --------- test/testutils/DataBlockFixture.h | 47 ----- test/utils/BlockStoreUtilsTest.cpp | 4 +- test/utils/DataTest.cpp | 170 ------------------ test/utils/KeyTest.cpp | 144 --------------- utils/BlockStoreUtils.cpp | 3 +- utils/Data.cpp | 100 ----------- utils/Data.h | 46 ----- utils/FixedSizeData.h | 112 ------------ utils/Key.h | 4 +- 45 files changed, 170 insertions(+), 855 deletions(-) delete mode 100644 test/testutils/DataBlockFixture.cpp delete mode 100644 test/testutils/DataBlockFixture.h delete mode 100644 test/utils/DataTest.cpp delete mode 100644 test/utils/KeyTest.cpp delete mode 100644 utils/Data.cpp delete mode 100644 utils/Data.h delete mode 100644 utils/FixedSizeData.h diff --git a/implementations/caching/CachingBlockStore.cpp b/implementations/caching/CachingBlockStore.cpp index 532e32c6..6518d80c 100644 --- a/implementations/caching/CachingBlockStore.cpp +++ b/implementations/caching/CachingBlockStore.cpp @@ -9,6 +9,7 @@ using std::unique_ptr; using std::make_unique; using cpputils::dynamic_pointer_move; +using cpputils::Data; namespace blockstore { namespace caching { diff --git a/implementations/caching/CachingBlockStore.h b/implementations/caching/CachingBlockStore.h index 49c67c27..ae298f24 100644 --- a/implementations/caching/CachingBlockStore.h +++ b/implementations/caching/CachingBlockStore.h @@ -14,14 +14,14 @@ public: CachingBlockStore(std::unique_ptr baseBlockStore); Key createKey() override; - std::unique_ptr tryCreate(const Key &key, Data data) override; + std::unique_ptr tryCreate(const Key &key, cpputils::Data data) override; std::unique_ptr load(const Key &key) override; void remove(std::unique_ptr block) override; uint64_t numBlocks() const override; void release(std::unique_ptr block); - std::unique_ptr tryCreateInBaseStore(const Key &key, Data data); + std::unique_ptr tryCreateInBaseStore(const Key &key, cpputils::Data data); void removeFromBaseStore(std::unique_ptr block); private: diff --git a/implementations/caching/NewBlock.cpp b/implementations/caching/NewBlock.cpp index edbfeb61..ee8837d3 100644 --- a/implementations/caching/NewBlock.cpp +++ b/implementations/caching/NewBlock.cpp @@ -3,6 +3,7 @@ using std::unique_ptr; using std::make_unique; +using cpputils::Data; namespace blockstore { namespace caching { diff --git a/implementations/caching/NewBlock.h b/implementations/caching/NewBlock.h index d71dbf69..5b2b69ac 100644 --- a/implementations/caching/NewBlock.h +++ b/implementations/caching/NewBlock.h @@ -3,7 +3,7 @@ #define BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_ #include "../../interface/BlockStore.h" -#include "../../utils/Data.h" +#include #include "messmer/cpp-utils/macros.h" #include @@ -20,7 +20,7 @@ class CachingBlockStore; // It only exists in the cache and it is created in the base block store when destructed. class NewBlock: public Block { public: - NewBlock(const Key &key, Data data, CachingBlockStore *blockStore); + NewBlock(const Key &key, cpputils::Data data, CachingBlockStore *blockStore); virtual ~NewBlock(); const void *data() const override; @@ -35,7 +35,7 @@ public: private: CachingBlockStore *_blockStore; - Data _data; + cpputils::Data _data; std::unique_ptr _baseBlock; bool _dataChanged; diff --git a/implementations/encrypted/EncryptedBlock.h b/implementations/encrypted/EncryptedBlock.h index cb0e10a4..2b17e843 100644 --- a/implementations/encrypted/EncryptedBlock.h +++ b/implementations/encrypted/EncryptedBlock.h @@ -3,7 +3,7 @@ #define BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_ #include "../../interface/Block.h" -#include "../../utils/Data.h" +#include #include "../../interface/BlockStore.h" #include "messmer/cpp-utils/macros.h" @@ -21,11 +21,11 @@ template class EncryptedBlock: public Block { public: BOOST_CONCEPT_ASSERT((CipherConcept)); - static std::unique_ptr TryCreateNew(BlockStore *baseBlockStore, const Key &key, Data data, const typename Cipher::EncryptionKey &encKey); + static std::unique_ptr TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey); static std::unique_ptr TryDecrypt(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key); //TODO Storing key twice (in parent class and in object pointed to). Once would be enough. - EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key, Data plaintextData); + EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key, cpputils::Data plaintextData); virtual ~EncryptedBlock(); const void *data() const override; @@ -38,7 +38,7 @@ public: private: std::unique_ptr _baseBlock; - Data _plaintextData; + cpputils::Data _plaintextData; typename Cipher::EncryptionKey _encKey; bool _dataChanged; @@ -50,8 +50,8 @@ private: template -std::unique_ptr> EncryptedBlock::TryCreateNew(BlockStore *baseBlockStore, const Key &key, Data data, const typename Cipher::EncryptionKey &encKey) { - Data encrypted = Cipher::encrypt((byte*)data.data(), data.size(), encKey); +std::unique_ptr> EncryptedBlock::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey) { + cpputils::Data encrypted = Cipher::encrypt((byte*)data.data(), data.size(), encKey); auto baseBlock = baseBlockStore->tryCreate(key, std::move(encrypted)); if (baseBlock.get() == nullptr) { //TODO Test this code branch @@ -64,7 +64,7 @@ std::unique_ptr> EncryptedBlock::TryCreateNew(Blo template std::unique_ptr> EncryptedBlock::TryDecrypt(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &encKey) { //TODO Change BlockStore so we can read their "class Data" objects instead of "void *data()", and then we can change the Cipher interface to take Data objects instead of "byte *" + size - boost::optional plaintext = Cipher::decrypt((byte*)baseBlock->data(), baseBlock->size(), encKey); + boost::optional plaintext = Cipher::decrypt((byte*)baseBlock->data(), baseBlock->size(), encKey); if(!plaintext) { return nullptr; } @@ -72,7 +72,7 @@ std::unique_ptr> EncryptedBlock::TryDecrypt(std:: } template -EncryptedBlock::EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &encKey, Data plaintextData) +EncryptedBlock::EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &encKey, cpputils::Data plaintextData) :Block(baseBlock->key()), _baseBlock(std::move(baseBlock)), _plaintextData(std::move(plaintextData)), @@ -111,7 +111,7 @@ size_t EncryptedBlock::size() const { template void EncryptedBlock::_encryptToBaseBlock() { if (_dataChanged) { - Data encrypted = Cipher::encrypt((byte*)_plaintextData.data(), _plaintextData.size(), _encKey); + cpputils::Data encrypted = Cipher::encrypt((byte*)_plaintextData.data(), _plaintextData.size(), _encKey); _baseBlock->write(encrypted.data(), 0, encrypted.size()); _dataChanged = false; } diff --git a/implementations/encrypted/EncryptedBlockStore.h b/implementations/encrypted/EncryptedBlockStore.h index c116200d..9fb01c84 100644 --- a/implementations/encrypted/EncryptedBlockStore.h +++ b/implementations/encrypted/EncryptedBlockStore.h @@ -17,7 +17,7 @@ public: EncryptedBlockStore(std::unique_ptr baseBlockStore, const typename Cipher::EncryptionKey &encKey); Key createKey() override; - std::unique_ptr tryCreate(const Key &key, Data data) override; + std::unique_ptr tryCreate(const Key &key, cpputils::Data data) override; std::unique_ptr load(const Key &key) override; void remove(std::unique_ptr block) override; uint64_t numBlocks() const override; @@ -42,7 +42,7 @@ Key EncryptedBlockStore::createKey() { } template -std::unique_ptr EncryptedBlockStore::tryCreate(const Key &key, Data data) { +std::unique_ptr EncryptedBlockStore::tryCreate(const Key &key, cpputils::Data data) { return EncryptedBlock::TryCreateNew(_baseBlockStore.get(), key, std::move(data), _encKey); } diff --git a/implementations/encrypted/ciphers/AES256_CFB.cpp b/implementations/encrypted/ciphers/AES256_CFB.cpp index 81a3ea41..63d93ac7 100644 --- a/implementations/encrypted/ciphers/AES256_CFB.cpp +++ b/implementations/encrypted/ciphers/AES256_CFB.cpp @@ -3,6 +3,8 @@ using CryptoPP::CFB_Mode; using CryptoPP::AES; +using cpputils::Data; +using cpputils::FixedSizeData; namespace blockstore { namespace encrypted { diff --git a/implementations/encrypted/ciphers/AES256_CFB.h b/implementations/encrypted/ciphers/AES256_CFB.h index 39f36129..313842d6 100644 --- a/implementations/encrypted/ciphers/AES256_CFB.h +++ b/implementations/encrypted/ciphers/AES256_CFB.h @@ -2,8 +2,8 @@ #ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHERS_AES256_CFB_H_ #define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHERS_AES256_CFB_H_ -#include "../../../utils/FixedSizeData.h" -#include "../../../utils/Data.h" +#include +#include #include #include #include "Cipher.h" @@ -15,7 +15,7 @@ class AES256_CFB { public: BOOST_CONCEPT_ASSERT((CipherConcept)); - using EncryptionKey = FixedSizeData<32>; + using EncryptionKey = cpputils::FixedSizeData<32>; static_assert(32 == CryptoPP::AES::MAX_KEYLENGTH, "If AES offered larger keys, we should offer a variant with it"); static constexpr unsigned int ciphertextSize(unsigned int plaintextBlockSize) { @@ -26,8 +26,8 @@ public: return ciphertextBlockSize - IV_SIZE; } - static Data encrypt(const byte *plaintext, unsigned int plaintextSize, const EncryptionKey &encKey); - static boost::optional decrypt(const byte *ciphertext, unsigned int ciphertextSize, const EncryptionKey &encKey); + static cpputils::Data encrypt(const byte *plaintext, unsigned int plaintextSize, const EncryptionKey &encKey); + static boost::optional decrypt(const byte *ciphertext, unsigned int ciphertextSize, const EncryptionKey &encKey); private: static constexpr unsigned int IV_SIZE = CryptoPP::AES::BLOCKSIZE; diff --git a/implementations/encrypted/ciphers/AES256_GCM.cpp b/implementations/encrypted/ciphers/AES256_GCM.cpp index 6118977c..d985c786 100644 --- a/implementations/encrypted/ciphers/AES256_GCM.cpp +++ b/implementations/encrypted/ciphers/AES256_GCM.cpp @@ -9,6 +9,8 @@ using CryptoPP::ArraySource; using CryptoPP::ArraySink; using CryptoPP::GCM_64K_Tables; using CryptoPP::HashVerificationFilter; +using cpputils::Data; +using cpputils::FixedSizeData; namespace blockstore { namespace encrypted { diff --git a/implementations/encrypted/ciphers/AES256_GCM.h b/implementations/encrypted/ciphers/AES256_GCM.h index f13c2a03..014ef1f0 100644 --- a/implementations/encrypted/ciphers/AES256_GCM.h +++ b/implementations/encrypted/ciphers/AES256_GCM.h @@ -2,10 +2,9 @@ #ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHERS_AES256_GCM_H_ #define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_CIPHERS_AES256_GCM_H_ -#include "../../../utils/FixedSizeData.h" -#include "../../../utils/Data.h" +#include +#include #include -#include #include "Cipher.h" namespace blockstore { @@ -15,7 +14,7 @@ class AES256_GCM { public: BOOST_CONCEPT_ASSERT((CipherConcept)); - using EncryptionKey = FixedSizeData<32>; + using EncryptionKey = cpputils::FixedSizeData<32>; static_assert(32 == CryptoPP::AES::MAX_KEYLENGTH, "If AES offered larger keys, we should offer a variant with it"); static constexpr unsigned int ciphertextSize(unsigned int plaintextBlockSize) { @@ -26,8 +25,8 @@ public: return ciphertextBlockSize - IV_SIZE - TAG_SIZE; } - static Data encrypt(const byte *plaintext, unsigned int plaintextSize, const EncryptionKey &encKey); - static boost::optional decrypt(const byte *ciphertext, unsigned int ciphertextSize, const EncryptionKey &encKey); + static cpputils::Data encrypt(const byte *plaintext, unsigned int plaintextSize, const EncryptionKey &encKey); + static boost::optional decrypt(const byte *ciphertext, unsigned int ciphertextSize, const EncryptionKey &encKey); private: static constexpr unsigned int IV_SIZE = CryptoPP::AES::BLOCKSIZE; diff --git a/implementations/encrypted/ciphers/Cipher.h b/implementations/encrypted/ciphers/Cipher.h index 08936427..5dddf4fc 100644 --- a/implementations/encrypted/ciphers/Cipher.h +++ b/implementations/encrypted/ciphers/Cipher.h @@ -15,8 +15,8 @@ public: same_type(UINT32_C(0), X::ciphertextSize(UINT32_C(5))); same_type(UINT32_C(0), X::plaintextSize(UINT32_C(5))); typename X::EncryptionKey key = X::EncryptionKey::CreateRandom(); - same_type(Data(0), X::encrypt((byte*)nullptr, UINT32_C(0), key)); - same_type(boost::optional(Data(0)), X::decrypt((byte*)nullptr, UINT32_C(0), key)); + same_type(cpputils::Data(0), X::encrypt((byte*)nullptr, UINT32_C(0), key)); + same_type(boost::optional(cpputils::Data(0)), X::decrypt((byte*)nullptr, UINT32_C(0), key)); } private: diff --git a/implementations/inmemory/InMemoryBlock.cpp b/implementations/inmemory/InMemoryBlock.cpp index 69d8a142..14d27df9 100644 --- a/implementations/inmemory/InMemoryBlock.cpp +++ b/implementations/inmemory/InMemoryBlock.cpp @@ -9,6 +9,7 @@ using std::ostream; using std::ifstream; using std::ofstream; using std::ios; +using cpputils::Data; namespace blockstore { namespace inmemory { diff --git a/implementations/inmemory/InMemoryBlock.h b/implementations/inmemory/InMemoryBlock.h index 4efb1553..bab8427c 100644 --- a/implementations/inmemory/InMemoryBlock.h +++ b/implementations/inmemory/InMemoryBlock.h @@ -3,7 +3,7 @@ #define BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_ #include "../../interface/Block.h" -#include "../../utils/Data.h" +#include namespace blockstore { namespace inmemory { @@ -11,7 +11,7 @@ class InMemoryBlockStore; class InMemoryBlock: public Block { public: - InMemoryBlock(const Key &key, Data size); + InMemoryBlock(const Key &key, cpputils::Data size); InMemoryBlock(const InMemoryBlock &rhs); virtual ~InMemoryBlock(); @@ -23,7 +23,7 @@ public: size_t size() const override; private: - std::shared_ptr _data; + std::shared_ptr _data; }; } diff --git a/implementations/inmemory/InMemoryBlockStore.cpp b/implementations/inmemory/InMemoryBlockStore.cpp index 3692b965..b4446349 100644 --- a/implementations/inmemory/InMemoryBlockStore.cpp +++ b/implementations/inmemory/InMemoryBlockStore.cpp @@ -9,6 +9,7 @@ using std::mutex; using std::lock_guard; using std::piecewise_construct; using std::make_tuple; +using cpputils::Data; namespace blockstore { namespace inmemory { diff --git a/implementations/inmemory/InMemoryBlockStore.h b/implementations/inmemory/InMemoryBlockStore.h index 3faea85f..b9ec5d25 100644 --- a/implementations/inmemory/InMemoryBlockStore.h +++ b/implementations/inmemory/InMemoryBlockStore.h @@ -16,7 +16,7 @@ class InMemoryBlockStore: public BlockStoreWithRandomKeys { public: InMemoryBlockStore(); - std::unique_ptr tryCreate(const Key &key, Data data) override; + std::unique_ptr tryCreate(const Key &key, cpputils::Data data) override; std::unique_ptr load(const Key &key) override; void remove(std::unique_ptr block) override; uint64_t numBlocks() const override; diff --git a/implementations/ondisk/OnDiskBlock.cpp b/implementations/ondisk/OnDiskBlock.cpp index 4c0f25ef..6fb621d2 100644 --- a/implementations/ondisk/OnDiskBlock.cpp +++ b/implementations/ondisk/OnDiskBlock.cpp @@ -13,6 +13,7 @@ using std::ostream; using std::ifstream; using std::ofstream; using std::ios; +using cpputils::Data; namespace bf = boost::filesystem; @@ -47,11 +48,15 @@ unique_ptr OnDiskBlock::LoadFromDisk(const bf::path &rootdir, const //If it isn't a file, Data::LoadFromFile() would usually also crash. We still need this extra check //upfront, because Data::LoadFromFile() doesn't crash if we give it the path of a directory //instead the path of a file. + //TODO Data::LoadFromFile now returns boost::optional. Do we then still need this? if(!bf::is_regular_file(filepath)) { return nullptr; } - Data data = Data::LoadFromFile(filepath); - return unique_ptr(new OnDiskBlock(key, filepath, std::move(data))); + boost::optional data = Data::LoadFromFile(filepath); + if (!data) { + return nullptr; + } + return unique_ptr(new OnDiskBlock(key, filepath, std::move(*data))); } catch (const FileDoesntExistException &e) { return nullptr; } diff --git a/implementations/ondisk/OnDiskBlock.h b/implementations/ondisk/OnDiskBlock.h index a139249a..3acea481 100644 --- a/implementations/ondisk/OnDiskBlock.h +++ b/implementations/ondisk/OnDiskBlock.h @@ -4,7 +4,7 @@ #include #include "../../interface/Block.h" -#include "../../utils/Data.h" +#include #include #include "messmer/cpp-utils/macros.h" @@ -18,7 +18,7 @@ public: virtual ~OnDiskBlock(); static std::unique_ptr LoadFromDisk(const boost::filesystem::path &rootdir, const Key &key); - static std::unique_ptr CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, Data data); + static std::unique_ptr CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data); static void RemoveFromDisk(const boost::filesystem::path &rootdir, const Key &key); const void *data() const override; @@ -30,10 +30,10 @@ public: private: const boost::filesystem::path _filepath; - Data _data; + cpputils::Data _data; bool _dataChanged; - OnDiskBlock(const Key &key, const boost::filesystem::path &filepath, Data data); + OnDiskBlock(const Key &key, const boost::filesystem::path &filepath, cpputils::Data data); void _fillDataWithZeroes(); void _storeToDisk() const; diff --git a/implementations/ondisk/OnDiskBlockStore.cpp b/implementations/ondisk/OnDiskBlockStore.cpp index 4757807b..3335a047 100644 --- a/implementations/ondisk/OnDiskBlockStore.cpp +++ b/implementations/ondisk/OnDiskBlockStore.cpp @@ -4,6 +4,7 @@ using std::unique_ptr; using std::make_unique; using std::string; +using cpputils::Data; namespace bf = boost::filesystem; diff --git a/implementations/ondisk/OnDiskBlockStore.h b/implementations/ondisk/OnDiskBlockStore.h index 3a668d4d..98ba7163 100644 --- a/implementations/ondisk/OnDiskBlockStore.h +++ b/implementations/ondisk/OnDiskBlockStore.h @@ -14,7 +14,7 @@ class OnDiskBlockStore: public BlockStoreWithRandomKeys { public: OnDiskBlockStore(const boost::filesystem::path &rootdir); - std::unique_ptr tryCreate(const Key &key, Data data) override; + std::unique_ptr tryCreate(const Key &key, cpputils::Data data) override; std::unique_ptr load(const Key &key) override; void remove(std::unique_ptr block) override; uint64_t numBlocks() const override; diff --git a/implementations/parallelaccess/ParallelAccessBlockStore.cpp b/implementations/parallelaccess/ParallelAccessBlockStore.cpp index 2f59bb79..19fdb343 100644 --- a/implementations/parallelaccess/ParallelAccessBlockStore.cpp +++ b/implementations/parallelaccess/ParallelAccessBlockStore.cpp @@ -24,7 +24,7 @@ Key ParallelAccessBlockStore::createKey() { return _baseBlockStore->createKey(); } -unique_ptr ParallelAccessBlockStore::tryCreate(const Key &key, Data data) { +unique_ptr ParallelAccessBlockStore::tryCreate(const Key &key, cpputils::Data data) { auto block = _baseBlockStore->tryCreate(key, std::move(data)); if (block.get() == nullptr) { //TODO Test this code branch diff --git a/implementations/parallelaccess/ParallelAccessBlockStore.h b/implementations/parallelaccess/ParallelAccessBlockStore.h index 7c7c7adf..8ba08ba6 100644 --- a/implementations/parallelaccess/ParallelAccessBlockStore.h +++ b/implementations/parallelaccess/ParallelAccessBlockStore.h @@ -15,7 +15,7 @@ public: ParallelAccessBlockStore(std::unique_ptr baseBlockStore); Key createKey() override; - std::unique_ptr tryCreate(const Key &key, Data data) override; + std::unique_ptr tryCreate(const Key &key, cpputils::Data data) override; std::unique_ptr load(const Key &key) override; void remove(std::unique_ptr block) override; uint64_t numBlocks() const override; diff --git a/implementations/testfake/FakeBlock.cpp b/implementations/testfake/FakeBlock.cpp index adf8bcdf..b6505aa7 100644 --- a/implementations/testfake/FakeBlock.cpp +++ b/implementations/testfake/FakeBlock.cpp @@ -10,6 +10,7 @@ using std::ifstream; using std::ofstream; using std::ios; using std::string; +using cpputils::Data; namespace blockstore { namespace testfake { diff --git a/implementations/testfake/FakeBlock.h b/implementations/testfake/FakeBlock.h index 7f49f224..a0fc5c41 100644 --- a/implementations/testfake/FakeBlock.h +++ b/implementations/testfake/FakeBlock.h @@ -3,7 +3,7 @@ #define BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_ #include "../../interface/Block.h" -#include "../../utils/Data.h" +#include #include "messmer/cpp-utils/macros.h" @@ -13,7 +13,7 @@ class FakeBlockStore; class FakeBlock: public Block { public: - FakeBlock(FakeBlockStore *store, const Key &key, std::shared_ptr data, bool dirty); + FakeBlock(FakeBlockStore *store, const Key &key, std::shared_ptr data, bool dirty); virtual ~FakeBlock(); const void *data() const override; @@ -25,7 +25,7 @@ public: private: FakeBlockStore *_store; - std::shared_ptr _data; + std::shared_ptr _data; bool _dataChanged; DISALLOW_COPY_AND_ASSIGN(FakeBlock); diff --git a/implementations/testfake/FakeBlockStore.cpp b/implementations/testfake/FakeBlockStore.cpp index 50dce405..eb1e09f4 100644 --- a/implementations/testfake/FakeBlockStore.cpp +++ b/implementations/testfake/FakeBlockStore.cpp @@ -7,6 +7,7 @@ using std::make_shared; using std::string; using std::mutex; using std::lock_guard; +using cpputils::Data; namespace blockstore { namespace testfake { diff --git a/implementations/testfake/FakeBlockStore.h b/implementations/testfake/FakeBlockStore.h index cec32b56..9d291cec 100644 --- a/implementations/testfake/FakeBlockStore.h +++ b/implementations/testfake/FakeBlockStore.h @@ -3,7 +3,7 @@ #define BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_ #include "../../interface/helpers/BlockStoreWithRandomKeys.h" -#include "../../utils/Data.h" +#include #include "messmer/cpp-utils/macros.h" #include @@ -31,24 +31,24 @@ class FakeBlockStore: public BlockStoreWithRandomKeys { public: FakeBlockStore(); - std::unique_ptr tryCreate(const Key &key, Data data) override; + std::unique_ptr tryCreate(const Key &key, cpputils::Data data) override; std::unique_ptr load(const Key &key) override; void remove(std::unique_ptr block) override; uint64_t numBlocks() const override; - void updateData(const Key &key, const Data &data); + void updateData(const Key &key, const cpputils::Data &data); private: - std::map _blocks; + std::map _blocks; //This vector keeps a handle of the data regions for all created FakeBlock objects. //This way, it is ensured that no two created FakeBlock objects will work on the //same data region. Without this, it could happen that a test case creates a FakeBlock, //destructs it, creates another one, and the new one gets the same memory region. //We want to avoid this for the reasons mentioned above (overflow data). - std::vector> _used_dataregions_for_blocks; + std::vector> _used_dataregions_for_blocks; - std::unique_ptr makeFakeBlockFromData(const Key &key, const Data &data, bool dirty); + std::unique_ptr makeFakeBlockFromData(const Key &key, const cpputils::Data &data, bool dirty); DISALLOW_COPY_AND_ASSIGN(FakeBlockStore); }; diff --git a/interface/BlockStore.h b/interface/BlockStore.h index 74d8d59d..9bb9b14e 100644 --- a/interface/BlockStore.h +++ b/interface/BlockStore.h @@ -5,7 +5,7 @@ #include "Block.h" #include #include -#include "../utils/Data.h" +#include namespace blockstore { @@ -15,14 +15,14 @@ public: virtual Key createKey() = 0; //Returns nullptr if key already exists - virtual std::unique_ptr tryCreate(const Key &key, Data data) = 0; + virtual std::unique_ptr tryCreate(const Key &key, cpputils::Data data) = 0; //TODO Use boost::optional (if key doesn't exist) // Return nullptr if block with this key doesn't exists virtual std::unique_ptr load(const Key &key) = 0; virtual void remove(std::unique_ptr block) = 0; virtual uint64_t numBlocks() const = 0; - std::unique_ptr create(const Data &data) { + std::unique_ptr create(const cpputils::Data &data) { std::unique_ptr block(nullptr); while(block.get() == nullptr) { //TODO Copy necessary? diff --git a/test/implementations/encrypted/CipherTest.cpp b/test/implementations/encrypted/CipherTest.cpp index a505c7b3..ec84837b 100644 --- a/test/implementations/encrypted/CipherTest.cpp +++ b/test/implementations/encrypted/CipherTest.cpp @@ -3,13 +3,14 @@ #include "../../../implementations/encrypted/ciphers/AES256_GCM.h" #include "../../../implementations/encrypted/ciphers/Cipher.h" -#include "../../testutils/DataBlockFixture.h" -#include "../../../utils/Data.h" +#include +#include #include using namespace blockstore::encrypted; -using blockstore::Data; +using cpputils::Data; +using cpputils::DataBlockFixture; template class CipherTest: public ::testing::Test { diff --git a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp index 42663fde..0a36ebc4 100644 --- a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp +++ b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp @@ -2,17 +2,17 @@ #include "../../../../implementations/ondisk/OnDiskBlock.h" #include "google/gtest/gtest.h" -#include "messmer/tempfile/src/TempFile.h" -#include "messmer/tempfile/src/TempDir.h" +#include +#include using ::testing::Test; using ::testing::WithParamInterface; using ::testing::Values; -using tempfile::TempFile; -using tempfile::TempDir; - using std::unique_ptr; +using cpputils::Data; +using cpputils::TempFile; +using cpputils::TempDir; using namespace blockstore; using namespace blockstore::ondisk; @@ -63,12 +63,12 @@ public: INSTANTIATE_TEST_CASE_P(OnDiskBlockCreateSizeTest, OnDiskBlockCreateSizeTest, Values(0, 1, 5, 1024, 10*1024*1024)); TEST_P(OnDiskBlockCreateSizeTest, OnDiskSizeIsCorrect) { - Data fileContent = Data::LoadFromFile(file.path()); + Data fileContent = Data::LoadFromFile(file.path()).value(); EXPECT_EQ(GetParam(), fileContent.size()); } TEST_P(OnDiskBlockCreateSizeTest, OnDiskBlockIsZeroedOut) { - Data fileContent = Data::LoadFromFile(file.path()); + Data fileContent = Data::LoadFromFile(file.path()).value(); EXPECT_EQ(0, std::memcmp(ZEROES.data(), fileContent.data(), fileContent.size())); } diff --git a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp index d2877f0c..9ac4c602 100644 --- a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp +++ b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp @@ -1,19 +1,20 @@ #include "../../../../implementations/ondisk/FileAlreadyExistsException.h" #include "../../../../implementations/ondisk/OnDiskBlock.h" -#include "../../../testutils/DataBlockFixture.h" +#include #include "google/gtest/gtest.h" -#include "messmer/tempfile/src/TempFile.h" -#include "messmer/tempfile/src/TempDir.h" +#include +#include using ::testing::Test; using ::testing::WithParamInterface; using ::testing::Values; -using tempfile::TempFile; -using tempfile::TempDir; - using std::unique_ptr; +using cpputils::Data; +using cpputils::DataBlockFixture; +using cpputils::TempFile; +using cpputils::TempDir; using namespace blockstore; using namespace blockstore::ondisk; @@ -60,7 +61,7 @@ public: } void EXPECT_STORED_FILE_DATA_CORRECT() { - Data actual = Data::LoadFromFile(file.path()); + Data actual = Data::LoadFromFile(file.path()).value(); EXPECT_EQ(randomData.size(), actual.size()); EXPECT_EQ(0, std::memcmp(randomData.data(), actual.data(), randomData.size())); } diff --git a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp index 8a32794b..364a2033 100644 --- a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp +++ b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp @@ -1,23 +1,24 @@ #include "../../../../implementations/ondisk/OnDiskBlock.h" -#include "../../../testutils/DataBlockFixture.h" +#include #include "../../../../utils/FileDoesntExistException.h" #include "google/gtest/gtest.h" -#include "../../../../utils/Data.h" -#include "messmer/tempfile/src/TempFile.h" -#include "messmer/tempfile/src/TempDir.h" +#include +#include +#include #include using ::testing::Test; using ::testing::WithParamInterface; using ::testing::Values; -using tempfile::TempFile; -using tempfile::TempDir; - using std::ofstream; using std::unique_ptr; using std::ios; +using cpputils::Data; +using cpputils::DataBlockFixture; +using cpputils::TempFile; +using cpputils::TempDir; using namespace blockstore; using namespace blockstore::ondisk; diff --git a/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp b/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp index fa0109a6..78a8535b 100644 --- a/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp +++ b/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp @@ -1,7 +1,7 @@ #include "../../../interface/helpers/BlockStoreWithRandomKeys.h" #include "google/gtest/gtest.h" #include "google/gmock/gmock.h" -#include "../../testutils/DataBlockFixture.h" +#include using ::testing::Test; using ::testing::_; @@ -13,6 +13,8 @@ using ::testing::ByRef; using std::string; using std::unique_ptr; using std::make_unique; +using cpputils::Data; +using cpputils::DataBlockFixture; using namespace blockstore; diff --git a/test/testutils/BlockStoreTest.h b/test/testutils/BlockStoreTest.h index ad7d940c..f97eb4d5 100644 --- a/test/testutils/BlockStoreTest.h +++ b/test/testutils/BlockStoreTest.h @@ -4,8 +4,6 @@ #include "google/gtest/gtest.h" -#include "DataBlockFixture.h" - #include "../../interface/BlockStore.h" class BlockStoreTestFixture { @@ -28,14 +26,14 @@ TYPED_TEST_CASE_P(BlockStoreTest); TYPED_TEST_P(BlockStoreTest, TwoCreatedBlocksHaveDifferentKeys) { auto blockStore = this->fixture.createBlockStore(); - auto block1 = blockStore->create(blockstore::Data(1024)); - auto block2 = blockStore->create(blockstore::Data(1024)); + auto block1 = blockStore->create(cpputils::Data(1024)); + auto block2 = blockStore->create(cpputils::Data(1024)); EXPECT_NE(block1->key(), block2->key()); } TYPED_TEST_P(BlockStoreTest, BlockIsNotLoadableAfterDeleting) { auto blockStore = this->fixture.createBlockStore(); - auto blockkey = blockStore->create(blockstore::Data(1024))->key(); + auto blockkey = blockStore->create(cpputils::Data(1024))->key(); auto block = blockStore->load(blockkey); EXPECT_NE(nullptr, block.get()); blockStore->remove(std::move(block)); @@ -49,55 +47,55 @@ TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectOnEmptyBlockstore) { TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingOneBlock) { auto blockStore = this->fixture.createBlockStore(); - auto block = blockStore->create(blockstore::Data(1)); + auto block = blockStore->create(cpputils::Data(1)); EXPECT_EQ(1, blockStore->numBlocks()); } TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingOneBlock_AfterClosingBlock) { auto blockStore = this->fixture.createBlockStore(); - blockStore->create(blockstore::Data(1)); + blockStore->create(cpputils::Data(1)); EXPECT_EQ(1, blockStore->numBlocks()); } TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterRemovingTheLastBlock) { auto blockStore = this->fixture.createBlockStore(); - auto block = blockStore->create(blockstore::Data(1)); + auto block = blockStore->create(cpputils::Data(1)); blockStore->remove(std::move(block)); EXPECT_EQ(0, blockStore->numBlocks()); } TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks) { auto blockStore = this->fixture.createBlockStore(); - auto block1 = blockStore->create(blockstore::Data(1)); - auto block2 = blockStore->create(blockstore::Data(0)); + auto block1 = blockStore->create(cpputils::Data(1)); + auto block2 = blockStore->create(cpputils::Data(0)); EXPECT_EQ(2, blockStore->numBlocks()); } TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingFirstBlock) { auto blockStore = this->fixture.createBlockStore(); - blockStore->create(blockstore::Data(1)); - auto block2 = blockStore->create(blockstore::Data(0)); + blockStore->create(cpputils::Data(1)); + auto block2 = blockStore->create(cpputils::Data(0)); EXPECT_EQ(2, blockStore->numBlocks()); } TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingSecondBlock) { auto blockStore = this->fixture.createBlockStore(); - auto block1 = blockStore->create(blockstore::Data(1)); - blockStore->create(blockstore::Data(0)); + auto block1 = blockStore->create(cpputils::Data(1)); + blockStore->create(cpputils::Data(0)); EXPECT_EQ(2, blockStore->numBlocks()); } TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterAddingTwoBlocks_AfterClosingBothBlocks) { auto blockStore = this->fixture.createBlockStore(); - blockStore->create(blockstore::Data(1)); - blockStore->create(blockstore::Data(0)); + blockStore->create(cpputils::Data(1)); + blockStore->create(cpputils::Data(0)); EXPECT_EQ(2, blockStore->numBlocks()); } TYPED_TEST_P(BlockStoreTest, NumBlocksIsCorrectAfterRemovingABlock) { auto blockStore = this->fixture.createBlockStore(); - auto block = blockStore->create(blockstore::Data(1)); - blockStore->create(blockstore::Data(1)); + auto block = blockStore->create(cpputils::Data(1)); + blockStore->create(cpputils::Data(1)); blockStore->remove(std::move(block)); EXPECT_EQ(1, blockStore->numBlocks()); } diff --git a/test/testutils/BlockStoreTest_Data.h b/test/testutils/BlockStoreTest_Data.h index 11dfa2ac..30acbff1 100644 --- a/test/testutils/BlockStoreTest_Data.h +++ b/test/testutils/BlockStoreTest_Data.h @@ -13,14 +13,14 @@ public: : blockStore(std::move(blockStore_)), testData(testData_), foregroundData(testData.count), backgroundData(testData.blocksize) { - DataBlockFixture _foregroundData(testData.count); - DataBlockFixture _backgroundData(testData.blocksize); + cpputils::DataBlockFixture _foregroundData(testData.count); + cpputils::DataBlockFixture _backgroundData(testData.blocksize); std::memcpy(foregroundData.data(), _foregroundData.data(), foregroundData.size()); std::memcpy(backgroundData.data(), _backgroundData.data(), backgroundData.size()); } void TestWriteAndReadImmediately() { - auto block = blockStore->create(blockstore::Data(testData.blocksize).FillWithZeroes()); + auto block = blockStore->create(cpputils::Data(testData.blocksize).FillWithZeroes()); block->write(foregroundData.data(), testData.offset, testData.count); EXPECT_DATA_READS_AS(foregroundData, *block, testData.offset, testData.count); @@ -36,7 +36,7 @@ public: } void TestOverwriteAndRead() { - auto block = blockStore->create(blockstore::Data(testData.blocksize)); + auto block = blockStore->create(cpputils::Data(testData.blocksize)); block->write(backgroundData.data(), 0, testData.blocksize); block->write(foregroundData.data(), testData.offset, testData.count); EXPECT_DATA_READS_AS(foregroundData, *block, testData.offset, testData.count); @@ -46,30 +46,30 @@ public: private: std::unique_ptr blockStore; DataRange testData; - blockstore::Data foregroundData; - blockstore::Data backgroundData; + cpputils::Data foregroundData; + cpputils::Data backgroundData; - void EXPECT_DATA_EQ(const blockstore::Data &expected, const blockstore::Data &actual) { + void EXPECT_DATA_EQ(const cpputils::Data &expected, const cpputils::Data &actual) { EXPECT_EQ(expected.size(), actual.size()); EXPECT_EQ(0, std::memcmp(expected.data(), actual.data(), expected.size())); } - blockstore::Key CreateBlockWriteToItAndReturnKey(const blockstore::Data &to_write) { - auto newblock = blockStore->create(blockstore::Data(testData.blocksize).FillWithZeroes()); + blockstore::Key CreateBlockWriteToItAndReturnKey(const cpputils::Data &to_write) { + auto newblock = blockStore->create(cpputils::Data(testData.blocksize).FillWithZeroes()); newblock->write(to_write.data(), testData.offset, testData.count); return newblock->key(); } - void EXPECT_DATA_READS_AS(const blockstore::Data &expected, const blockstore::Block &block, off_t offset, size_t count) { - blockstore::Data read(count); + void EXPECT_DATA_READS_AS(const cpputils::Data &expected, const blockstore::Block &block, off_t offset, size_t count) { + cpputils::Data read(count); std::memcpy(read.data(), (uint8_t*)block.data() + offset, count); EXPECT_DATA_EQ(expected, read); } - void EXPECT_DATA_READS_AS_OUTSIDE_OF(const blockstore::Data &expected, const blockstore::Block &block, off_t start, size_t count) { - blockstore::Data begin(start); - blockstore::Data end(testData.blocksize - count - start); + void EXPECT_DATA_READS_AS_OUTSIDE_OF(const cpputils::Data &expected, const blockstore::Block &block, off_t start, size_t count) { + cpputils::Data begin(start); + cpputils::Data end(testData.blocksize - count - start); std::memcpy(begin.data(), expected.data(), start); std::memcpy(end.data(), (uint8_t*)expected.data()+start+count, end.size()); @@ -79,7 +79,7 @@ private: } void EXPECT_DATA_IS_ZEROES_OUTSIDE_OF(const blockstore::Block &block, off_t start, size_t count) { - blockstore::Data ZEROES(testData.blocksize); + cpputils::Data ZEROES(testData.blocksize); ZEROES.FillWithZeroes(); EXPECT_DATA_READS_AS_OUTSIDE_OF(ZEROES, block, start, count); } diff --git a/test/testutils/BlockStoreTest_Size.h b/test/testutils/BlockStoreTest_Size.h index 5948d836..02679e20 100644 --- a/test/testutils/BlockStoreTest_Size.h +++ b/test/testutils/BlockStoreTest_Size.h @@ -1,6 +1,7 @@ // This file is meant to be included by BlockStoreTest.h only -#include "../../utils/Data.h" +#include +#include class BlockStoreSizeParameterizedTest { public: @@ -18,8 +19,8 @@ public: } void TestCreatedBlockData() { - DataBlockFixture dataFixture(size); - blockstore::Data data(size); + cpputils::DataBlockFixture dataFixture(size); + cpputils::Data data(size); std::memcpy(data.data(), dataFixture.data(), size); auto block = blockStore->create(data); EXPECT_EQ(0, std::memcmp(dataFixture.data(), block->data(), size)); @@ -27,8 +28,8 @@ public: } void TestLoadingUnchangedBlockData() { - DataBlockFixture dataFixture(size); - blockstore::Data data(size); + cpputils::DataBlockFixture dataFixture(size); + cpputils::Data data(size); std::memcpy(data.data(), dataFixture.data(), size); blockstore::Key key = blockStore->create(data)->key(); auto loaded_block = blockStore->load(key); @@ -36,21 +37,21 @@ public: } void TestLoadedBlockIsCorrect() { - DataBlockFixture randomData(size); + cpputils::DataBlockFixture randomData(size); auto loaded_block = StoreDataToBlockAndLoadIt(randomData); EXPECT_EQ(size, loaded_block->size()); EXPECT_EQ(0, std::memcmp(randomData.data(), loaded_block->data(), size)); } void TestLoadedBlockIsCorrectWhenLoadedDirectlyAfterFlushing() { - DataBlockFixture randomData(size); + cpputils::DataBlockFixture randomData(size); auto loaded_block = StoreDataToBlockAndLoadItDirectlyAfterFlushing(randomData); EXPECT_EQ(size, loaded_block->size()); EXPECT_EQ(0, std::memcmp(randomData.data(), loaded_block->data(), size)); } void TestAfterCreate_FlushingDoesntChangeBlock() { - DataBlockFixture randomData(size); + cpputils::DataBlockFixture randomData(size); auto block = CreateBlock(); WriteDataToBlock(block.get(), randomData); block->flush(); @@ -59,7 +60,7 @@ public: } void TestAfterLoad_FlushingDoesntChangeBlock() { - DataBlockFixture randomData(size); + cpputils::DataBlockFixture randomData(size); auto block = CreateBlockAndLoadIt(); WriteDataToBlock(block.get(), randomData); block->flush(); @@ -68,10 +69,10 @@ public: } void TestAfterCreate_FlushesWhenDestructed() { - DataBlockFixture randomData(size); + cpputils::DataBlockFixture randomData(size); blockstore::Key key = key; { - auto block = blockStore->create(blockstore::Data(size)); + auto block = blockStore->create(cpputils::Data(size)); key = block->key(); WriteDataToBlock(block.get(), randomData); } @@ -80,7 +81,7 @@ public: } void TestAfterLoad_FlushesWhenDestructed() { - DataBlockFixture randomData(size); + cpputils::DataBlockFixture randomData(size); blockstore::Key key = key; { key = CreateBlock()->key(); @@ -102,25 +103,25 @@ private: std::unique_ptr blockStore; size_t size; - blockstore::Data ZEROES(size_t size) { - blockstore::Data ZEROES(size); + cpputils::Data ZEROES(size_t size) { + cpputils::Data ZEROES(size); ZEROES.FillWithZeroes(); return ZEROES; } - std::unique_ptr StoreDataToBlockAndLoadIt(const DataBlockFixture &data) { + std::unique_ptr StoreDataToBlockAndLoadIt(const cpputils::DataBlockFixture &data) { blockstore::Key key = StoreDataToBlockAndGetKey(data); return blockStore->load(key); } - blockstore::Key StoreDataToBlockAndGetKey(const DataBlockFixture &dataFixture) { - blockstore::Data data(dataFixture.size()); + blockstore::Key StoreDataToBlockAndGetKey(const cpputils::DataBlockFixture &dataFixture) { + cpputils::Data data(dataFixture.size()); std::memcpy(data.data(), dataFixture.data(), dataFixture.size()); return blockStore->create(data)->key(); } - std::unique_ptr StoreDataToBlockAndLoadItDirectlyAfterFlushing(const DataBlockFixture &dataFixture) { - blockstore::Data data(dataFixture.size()); + std::unique_ptr StoreDataToBlockAndLoadItDirectlyAfterFlushing(const cpputils::DataBlockFixture &dataFixture) { + cpputils::Data data(dataFixture.size()); std::memcpy(data.data(), dataFixture.data(), dataFixture.size()); auto block = blockStore->create(data); block->flush(); @@ -133,14 +134,14 @@ private: } std::unique_ptr CreateBlock() { - return blockStore->create(blockstore::Data(size)); + return blockStore->create(cpputils::Data(size)); } - void WriteDataToBlock(blockstore::Block *block, const DataBlockFixture &randomData) { + void WriteDataToBlock(blockstore::Block *block, const cpputils::DataBlockFixture &randomData) { block->write(randomData.data(), 0, randomData.size()); } - void EXPECT_BLOCK_DATA_CORRECT(const blockstore::Block &block, const DataBlockFixture &randomData) { + void EXPECT_BLOCK_DATA_CORRECT(const blockstore::Block &block, const cpputils::DataBlockFixture &randomData) { EXPECT_EQ(randomData.size(), block.size()); EXPECT_EQ(0, std::memcmp(randomData.data(), block.data(), randomData.size())); } diff --git a/test/testutils/BlockStoreWithRandomKeysTest.h b/test/testutils/BlockStoreWithRandomKeysTest.h index 03d64cab..3f2db103 100644 --- a/test/testutils/BlockStoreWithRandomKeysTest.h +++ b/test/testutils/BlockStoreWithRandomKeysTest.h @@ -3,7 +3,6 @@ #define TEST_BLOCKSTORE_IMPLEMENTATIONS_TESTUTILS_BLOCKSTOREWITHRANDOMKEYSTEST_H_ #include -#include "DataBlockFixture.h" #include "../../interface/BlockStore.h" @@ -31,45 +30,45 @@ TYPED_TEST_CASE_P(BlockStoreWithRandomKeysTest); TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndSameSize) { auto blockStore = this->fixture.createBlockStore(); - auto block = blockStore->tryCreate(this->key, blockstore::Data(1024)); + auto block = blockStore->tryCreate(this->key, cpputils::Data(1024)); block->flush(); - auto block2 = blockStore->tryCreate(this->key, blockstore::Data(1024)); + auto block2 = blockStore->tryCreate(this->key, cpputils::Data(1024)); EXPECT_TRUE((bool)block); EXPECT_FALSE((bool)block2); } TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndDifferentSize) { auto blockStore = this->fixture.createBlockStore(); - auto block = blockStore->tryCreate(this->key, blockstore::Data(1024)); + auto block = blockStore->tryCreate(this->key, cpputils::Data(1024)); block->flush(); - auto block2 = blockStore->tryCreate(this->key, blockstore::Data(4096)); + auto block2 = blockStore->tryCreate(this->key, cpputils::Data(4096)); EXPECT_TRUE((bool)block); EXPECT_FALSE((bool)block2); } TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndFirstNullSize) { auto blockStore = this->fixture.createBlockStore(); - auto block = blockStore->tryCreate(this->key, blockstore::Data(0)); + auto block = blockStore->tryCreate(this->key, cpputils::Data(0)); block->flush(); - auto block2 = blockStore->tryCreate(this->key, blockstore::Data(1024)); + auto block2 = blockStore->tryCreate(this->key, cpputils::Data(1024)); EXPECT_TRUE((bool)block); EXPECT_FALSE((bool)block2); } TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndSecondNullSize) { auto blockStore = this->fixture.createBlockStore(); - auto block = blockStore->tryCreate(this->key, blockstore::Data(1024)); + auto block = blockStore->tryCreate(this->key, cpputils::Data(1024)); block->flush(); - auto block2 = blockStore->tryCreate(this->key, blockstore::Data(0)); + auto block2 = blockStore->tryCreate(this->key, cpputils::Data(0)); EXPECT_TRUE((bool)block); EXPECT_FALSE((bool)block2); } TYPED_TEST_P(BlockStoreWithRandomKeysTest, CreateTwoBlocksWithSameKeyAndBothNullSize) { auto blockStore = this->fixture.createBlockStore(); - auto block = blockStore->tryCreate(this->key, blockstore::Data(0)); + auto block = blockStore->tryCreate(this->key, cpputils::Data(0)); block->flush(); - auto block2 = blockStore->tryCreate(this->key, blockstore::Data(0)); + auto block2 = blockStore->tryCreate(this->key, cpputils::Data(0)); EXPECT_TRUE((bool)block); EXPECT_FALSE((bool)block2); } diff --git a/test/testutils/DataBlockFixture.cpp b/test/testutils/DataBlockFixture.cpp deleted file mode 100644 index 252015b2..00000000 --- a/test/testutils/DataBlockFixture.cpp +++ /dev/null @@ -1,87 +0,0 @@ -#include "DataBlockFixture.h" -#include -#include - -using std::min; - -DataBlockFixture::DataBlockFixture(size_t size, long long int IV): _fileData(new char[size]), _size(size) { - fillFileWithRandomData(IV); -} - -DataBlockFixture::~DataBlockFixture() { - delete[] _fileData; -} - -void DataBlockFixture::fillFileWithRandomData(long long int IV) { - long long int val = IV; - for(size_t i=0; i<_size/sizeof(long long int); ++i) { - //MMIX linear congruential generator - val *= 6364136223846793005L; - val += 1442695040888963407; - reinterpret_cast(_fileData)[i] = val; - } - uint64_t alreadyWritten = (_size/sizeof(long long int))*sizeof(long long int); - val *= 6364136223846793005L; - val += 1442695040888963407; - char *remainingBytes = reinterpret_cast(&val); - //Fill remaining bytes - for(size_t i=0; i<_size-alreadyWritten; ++i) { - reinterpret_cast(_fileData)[alreadyWritten + i] = remainingBytes[i]; - } -} - -const char *DataBlockFixture::data() const { - return _fileData; -} - -int DataBlockFixture::read(void *buf, size_t count, off_t offset) { - size_t realCount = min(count, _size - offset); - memcpy(buf, _fileData+offset, realCount); - return realCount; -} - -size_t DataBlockFixture::size() const { - return _size; -} - -bool DataBlockFixture::fileContentEqual(const char *content, size_t count, off_t offset) { - return 0 == memcmp(content, _fileData + offset, count); -} - -DataBlockFixtureWriteable::DataBlockFixtureWriteable(size_t size, long long int IV) - :DataBlockFixture(size, IV), _originalSize(size) { - _originalFileData = new char[size]; - memcpy(_originalFileData, _fileData, size); -} - -DataBlockFixtureWriteable::~DataBlockFixtureWriteable() { - delete[] _originalFileData; -} - -void DataBlockFixtureWriteable::write(const void *buf, size_t count, off_t offset) { - extendFileSizeIfNecessary(count + offset); - - memcpy(_fileData+offset, buf, count); -} - -void DataBlockFixtureWriteable::extendFileSizeIfNecessary(size_t size) { - if (size > _size) { - extendFileSize(size); - } -} - -void DataBlockFixtureWriteable::extendFileSize(size_t size) { - char *newfile = new char[size]; - memcpy(newfile, _fileData, _size); - delete[] _fileData; - _fileData = newfile; - _size = size; -} - -bool DataBlockFixtureWriteable::sizeUnchanged() { - return _size == _originalSize; -} - -bool DataBlockFixtureWriteable::regionUnchanged(off_t offset, size_t count) { - return 0 == memcmp(_fileData+offset, _originalFileData+offset, count); -} diff --git a/test/testutils/DataBlockFixture.h b/test/testutils/DataBlockFixture.h deleted file mode 100644 index 73d763d9..00000000 --- a/test/testutils/DataBlockFixture.h +++ /dev/null @@ -1,47 +0,0 @@ -#pragma once -#ifndef TEST_TESTUTILS_DATABLOCKFIXTURE_H_ -#define TEST_TESTUTILS_DATABLOCKFIXTURE_H_ - -#include - -class DataBlockFixture { -public: - DataBlockFixture(size_t size, long long int IV = 1); - virtual ~DataBlockFixture(); - - int read(void *buf, size_t count, off_t offset); - - // Return true, iff the given data is equal to the data of the file at the given offset. - bool fileContentEqual(const char *content, size_t count, off_t offset); - - const char *data() const; - - size_t size() const; - -protected: - char *_fileData; - size_t _size; - -private: - void fillFileWithRandomData(long long int IV); -}; - -class DataBlockFixtureWriteable: public DataBlockFixture { -public: - DataBlockFixtureWriteable(size_t size, long long int IV = 1); - virtual ~DataBlockFixtureWriteable(); - - void write(const void *buf, size_t count, off_t offset); - - bool sizeUnchanged(); - bool regionUnchanged(off_t offset, size_t count); - -private: - void extendFileSizeIfNecessary(size_t size); - void extendFileSize(size_t size); - - char *_originalFileData; - size_t _originalSize; -}; - -#endif diff --git a/test/utils/BlockStoreUtilsTest.cpp b/test/utils/BlockStoreUtilsTest.cpp index 93515636..4d0aca82 100644 --- a/test/utils/BlockStoreUtilsTest.cpp +++ b/test/utils/BlockStoreUtilsTest.cpp @@ -1,5 +1,5 @@ #include "../../implementations/testfake/FakeBlockStore.h" -#include "../testutils/DataBlockFixture.h" +#include #include "../../utils/BlockStoreUtils.h" #include "google/gtest/gtest.h" @@ -11,6 +11,8 @@ using ::testing::Values; using std::make_unique; using std::unique_ptr; +using cpputils::Data; +using cpputils::DataBlockFixture; using namespace blockstore; using namespace blockstore::utils; diff --git a/test/utils/DataTest.cpp b/test/utils/DataTest.cpp deleted file mode 100644 index 3601745a..00000000 --- a/test/utils/DataTest.cpp +++ /dev/null @@ -1,170 +0,0 @@ -#include "../testutils/DataBlockFixture.h" -#include "../../utils/Data.h" -#include "../../utils/FileDoesntExistException.h" -#include "google/gtest/gtest.h" - -#include "messmer/tempfile/src/TempFile.h" - -#include - -using ::testing::Test; -using ::testing::WithParamInterface; -using ::testing::Values; - -using tempfile::TempFile; - -using std::ifstream; -using std::ofstream; - -namespace bf = boost::filesystem; - -using namespace blockstore; - -class DataTest: public Test { -public: - bool DataIsZeroes(const Data &data) { - for (size_t i = 0; i != data.size(); ++ i) { - if (((char*)data.data())[i] != 0) { - return false; - } - } - return true; - } - - void FillData(const DataBlockFixture &fillData, Data *data) { - ASSERT_EQ(fillData.size(), data->size()); - std::memcpy(data->data(), fillData.data(), fillData.size()); - } - - void EXPECT_DATA_CORRECT(const DataBlockFixture &expectedData, const Data &data) { - ASSERT_EQ(expectedData.size(), data.size()); - EXPECT_EQ(0, std::memcmp(expectedData.data(), data.data(), expectedData.size())); - } -}; - -class DataTestWithSizeParam: public DataTest, public WithParamInterface { -public: - DataBlockFixture randomData; - - DataTestWithSizeParam(): randomData(GetParam()) {} - - void FillData(Data *data) { - DataTest::FillData(randomData, data); - } - - void StoreData(const bf::path &filepath) { - ofstream file(filepath.c_str(), std::ios::binary | std::ios::trunc); - file.write(randomData.data(), randomData.size()); - } - - void EXPECT_STORED_FILE_DATA_CORRECT(const bf::path &filepath) { - EXPECT_EQ(randomData.size(), bf::file_size(filepath)); - - ifstream file(filepath.c_str(), std::ios::binary); - char *read_data = new char[randomData.size()]; - file.read(read_data, randomData.size()); - - EXPECT_EQ(0, std::memcmp(randomData.data(), read_data, randomData.size())); - delete[] read_data; - } - - void EXPECT_DATA_CORRECT(const Data &data) { - DataTest::EXPECT_DATA_CORRECT(randomData, data); - } -}; - -INSTANTIATE_TEST_CASE_P(DataTestWithSizeParam, DataTestWithSizeParam, Values(0, 1, 2, 1024, 4096, 10*1024*1024)); - -// Working on a large data area without a crash is a good indicator that we -// are actually working on memory that was validly allocated for us. -TEST_P(DataTestWithSizeParam, WriteAndCheck) { - Data data(GetParam()); - - FillData(&data); - EXPECT_DATA_CORRECT(data); -} - -TEST_P(DataTestWithSizeParam, Size) { - Data data(GetParam()); - EXPECT_EQ(GetParam(), data.size()); -} - -TEST_P(DataTestWithSizeParam, CheckStoredFile) { - Data data(GetParam()); - FillData(&data); - - TempFile file; - data.StoreToFile(file.path()); - - EXPECT_STORED_FILE_DATA_CORRECT(file.path()); -} - -TEST_P(DataTestWithSizeParam, CheckLoadedData) { - TempFile file; - StoreData(file.path()); - - Data data = Data::LoadFromFile(file.path()); - - EXPECT_DATA_CORRECT(data); -} - -TEST_P(DataTestWithSizeParam, StoreDoesntChangeData) { - Data data(GetParam()); - FillData(&data); - - TempFile file; - data.StoreToFile(file.path()); - - EXPECT_DATA_CORRECT(data); -} - -TEST_P(DataTestWithSizeParam, StoreAndLoad) { - Data data(GetParam()); - FillData(&data); - - TempFile file; - data.StoreToFile(file.path()); - Data loaded_data = Data::LoadFromFile(file.path()); - - EXPECT_DATA_CORRECT(loaded_data); -} - -TEST_P(DataTestWithSizeParam, Copy) { - Data data(GetParam()); - FillData(&data); - - Data copy = data.copy(); - - EXPECT_DATA_CORRECT(copy); -} - -TEST_F(DataTest, InitializeWithZeroes) { - Data data(10*1024); - data.FillWithZeroes(); - EXPECT_TRUE(DataIsZeroes(data)); -} - -TEST_F(DataTest, FillModifiedDataWithZeroes) { - Data data(10*1024); - DataBlockFixture randomData(10*1024); - FillData(randomData, &data); - EXPECT_FALSE(DataIsZeroes(data)); - - data.FillWithZeroes(); - EXPECT_TRUE(DataIsZeroes(data)); -} - -//Needs 64bit for representation. This value isn't in the size param list, because the list is also used for read/write checks. -TEST_F(DataTest, LargesizeSize) { - size_t size = 10L*1024*1024*1024; - Data data(size); - EXPECT_EQ(size, data.size()); -} - -TEST_F(DataTest, LoadingNonexistingFile) { - TempFile file(false); // Pass false to constructor, so the tempfile is not created - EXPECT_THROW( - Data::LoadFromFile(file.path()), - FileDoesntExistException - ); -} diff --git a/test/utils/KeyTest.cpp b/test/utils/KeyTest.cpp deleted file mode 100644 index aa78d1ec..00000000 --- a/test/utils/KeyTest.cpp +++ /dev/null @@ -1,144 +0,0 @@ -#include "../testutils/DataBlockFixture.h" -#include "../../utils/Data.h" -#include "../../utils/Key.h" -#include "google/gtest/gtest.h" - - -using ::testing::Test; -using ::testing::WithParamInterface; -using ::testing::Values; - -using std::string; - -using namespace blockstore; - -//TODO Test blockstore::FixedSizeData instead of blockstore::Key - -class KeyTest: public Test { -public: - const string KEY1_AS_STRING = "1491BB4932A389EE14BC7090AC772972"; - const string KEY2_AS_STRING = "272EE5517627CFA147A971A8E6E747E0"; - - const DataBlockFixture KEY3_AS_BINARY; - const DataBlockFixture KEY4_AS_BINARY; - - KeyTest() : KEY3_AS_BINARY(Key::BINARY_LENGTH, 1), KEY4_AS_BINARY(Key::BINARY_LENGTH, 2) {} - - void EXPECT_DATA_EQ(const DataBlockFixture &expected, const Data &actual) { - EXPECT_EQ(expected.size(), actual.size()); - EXPECT_EQ(0, std::memcmp(expected.data(), actual.data(), expected.size())); - } -}; - -TEST_F(KeyTest, CanGenerateRandomKeysWithoutCrashing) { - Key result = Key::CreateRandom(); -} - -TEST_F(KeyTest, CreatedRandomKeysHaveCorrectLength) { - Key key = Key::CreateRandom(); - EXPECT_EQ(Key::STRING_LENGTH, key.ToString().size()); -} - -TEST_F(KeyTest, EqualsTrue) { - Key key1_1 = Key::FromString(KEY1_AS_STRING); - Key key1_2 = Key::FromString(KEY1_AS_STRING); - - EXPECT_TRUE(key1_1 == key1_2); - EXPECT_TRUE(key1_2 == key1_1); -} - -TEST_F(KeyTest, EqualsFalse) { - Key key1_1 = Key::FromString(KEY1_AS_STRING); - Key key2_1 = Key::FromString(KEY2_AS_STRING); - - EXPECT_FALSE(key1_1 == key2_1); - EXPECT_FALSE(key2_1 == key1_1); -} - -TEST_F(KeyTest, NotEqualsFalse) { - Key key1_1 = Key::FromString(KEY1_AS_STRING); - Key key1_2 = Key::FromString(KEY1_AS_STRING); - - EXPECT_FALSE(key1_1 != key1_2); - EXPECT_FALSE(key1_2 != key1_1); -} - -TEST_F(KeyTest, NotEqualsTrue) { - Key key1_1 = Key::FromString(KEY1_AS_STRING); - Key key2_1 = Key::FromString(KEY2_AS_STRING); - - EXPECT_TRUE(key1_1 != key2_1); - EXPECT_TRUE(key2_1 != key1_1); -} - -class KeyTestWithStringKeyParam: public KeyTest, public WithParamInterface {}; -INSTANTIATE_TEST_CASE_P(KeyTestWithStringKeyParam, KeyTestWithStringKeyParam, Values("2898B4B8A13CA63CBE0F0278CCE465DB", "6FFEBAD90C0DAA2B79628F0627CE9841")); - -TEST_P(KeyTestWithStringKeyParam, FromAndToString) { - Key key = Key::FromString(GetParam()); - EXPECT_EQ(GetParam(), key.ToString()); -} - -TEST_P(KeyTestWithStringKeyParam, ToAndFromString) { - Key key = Key::FromString(GetParam()); - Key key2 = Key::FromString(key.ToString()); - EXPECT_EQ(key, key2); -} - -class KeyTestWithBinaryKeyParam: public KeyTest, public WithParamInterface { -public: - static const DataBlockFixture VALUE1; - static const DataBlockFixture VALUE2; -}; -const DataBlockFixture KeyTestWithBinaryKeyParam::VALUE1(Key::BINARY_LENGTH, 3); -const DataBlockFixture KeyTestWithBinaryKeyParam::VALUE2(Key::BINARY_LENGTH, 4); -INSTANTIATE_TEST_CASE_P(KeyTestWithBinaryKeyParam, KeyTestWithBinaryKeyParam, Values(&KeyTestWithBinaryKeyParam::VALUE1, &KeyTestWithBinaryKeyParam::VALUE2)); - -TEST_P(KeyTestWithBinaryKeyParam, FromAndToBinary) { - Key key = Key::FromBinary((uint8_t*)GetParam()->data()); - Data keydata(Key::BINARY_LENGTH); - key.ToBinary(keydata.data()); - EXPECT_DATA_EQ(*GetParam(), keydata); -} - -TEST_P(KeyTestWithBinaryKeyParam, ToAndFromBinary) { - Key key = Key::FromBinary((uint8_t*)GetParam()->data()); - Data stored(Key::BINARY_LENGTH); - key.ToBinary(stored.data()); - Key loaded = Key::FromBinary(stored.data()); - EXPECT_EQ(key, loaded); -} - -class KeyTestWithKeyParam: public KeyTest, public WithParamInterface {}; -INSTANTIATE_TEST_CASE_P(KeyTestWithKeyParam, KeyTestWithKeyParam, Values(Key::FromString("2898B4B8A13CA63CBE0F0278CCE465DB"), Key::FromString("6FFEBAD90C0DAA2B79628F0627CE9841"))); - -TEST_P(KeyTestWithKeyParam, CopyConstructor) { - Key copy(GetParam()); - EXPECT_EQ(GetParam(), copy); -} - -TEST_F(KeyTest, CopyConstructorDoesntChangeSource) { - Key key1 = Key::FromString(KEY1_AS_STRING); - Key key2(key1); - EXPECT_EQ(KEY1_AS_STRING, key1.ToString()); -} - -TEST_P(KeyTestWithKeyParam, IsEqualAfterAssignment1) { - Key key2 = Key::FromString(KEY2_AS_STRING); - EXPECT_NE(GetParam(), key2); - key2 = GetParam(); - EXPECT_EQ(GetParam(), key2); -} - -TEST_F(KeyTest, AssignmentDoesntChangeSource) { - Key key1 = Key::FromString(KEY1_AS_STRING); - Key key2 = Key::FromString(KEY2_AS_STRING); - key2 = key1; - EXPECT_EQ(KEY1_AS_STRING, key1.ToString()); -} - -// This tests that a Key object is very lightweight -// (we will often pass keys around) -TEST_F(KeyTest, KeyIsLightweightObject) { - EXPECT_EQ(Key::BINARY_LENGTH, sizeof(Key)); -} diff --git a/utils/BlockStoreUtils.cpp b/utils/BlockStoreUtils.cpp index e6c33e2c..2b20cdef 100644 --- a/utils/BlockStoreUtils.cpp +++ b/utils/BlockStoreUtils.cpp @@ -1,10 +1,11 @@ #include "../interface/BlockStore.h" #include "BlockStoreUtils.h" -#include "Data.h" +#include #include #include using std::unique_ptr; +using cpputils::Data; namespace blockstore { namespace utils { diff --git a/utils/Data.cpp b/utils/Data.cpp deleted file mode 100644 index 4895f727..00000000 --- a/utils/Data.cpp +++ /dev/null @@ -1,100 +0,0 @@ -#include "Data.h" -#include "FileDoesntExistException.h" -#include -#include - -using std::istream; -using std::ofstream; -using std::ifstream; -using std::ios; -using std::unique_ptr; -using std::make_unique; - -namespace bf = boost::filesystem; - -namespace blockstore { - -Data::Data(size_t size) -: _size(size), _data(std::malloc(size)) { - if (nullptr == _data) { - throw std::bad_alloc(); - } -} - -Data::Data(Data &&rhs) -: _size(rhs._size), _data(rhs._data) { - // Make rhs invalid, so the memory doesn't get freed in its destructor. - rhs._data = nullptr; -} - -Data::~Data() { - std::free(_data); - _data = nullptr; -} - -Data Data::copy() const { - Data copy(_size); - std::memcpy(copy._data, _data, _size); - return copy; -} - -void *Data::data() { - return const_cast(const_cast(this)->data()); -} - -const void *Data::data() const { - return _data; -} - -size_t Data::size() const { - return _size; -} - -Data &Data::FillWithZeroes() { - std::memset(_data, 0, _size); - return *this; -} - -void Data::StoreToFile(const bf::path &filepath) const { - ofstream file(filepath.c_str(), ios::binary | ios::trunc); - file.write((const char*)_data, _size); -} - -Data Data::LoadFromFile(const bf::path &filepath) { - ifstream file(filepath.c_str(), ios::binary); - _assertFileExists(file, filepath); - size_t size = _getStreamSize(file); - - Data result(size); - result._readFromStream(file); - return result; -} - -void Data::_assertFileExists(const ifstream &file, const bf::path &filepath) { - if (!file.good()) { - throw FileDoesntExistException(filepath); - } -} - -size_t Data::_getStreamSize(istream &stream) { - auto current_pos = stream.tellg(); - - //Retrieve length - stream.seekg(0, stream.end); - auto endpos = stream.tellg(); - - //Restore old position - stream.seekg(current_pos, stream.beg); - - return endpos - current_pos; -} - -void Data::_readFromStream(istream &stream) { - stream.read((char*)_data, _size); -} - -bool operator==(const Data &lhs, const Data &rhs) { - return lhs.size() == rhs.size() && 0 == memcmp(lhs.data(), rhs.data(), lhs.size()); -} - -} diff --git a/utils/Data.h b/utils/Data.h deleted file mode 100644 index 54d5ccc7..00000000 --- a/utils/Data.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once -#ifndef BLOCKSTORE_IMPLEMENTATIONS_ONDISK_DATA_H_ -#define BLOCKSTORE_IMPLEMENTATIONS_ONDISK_DATA_H_ - -#include -#include -#include -#include - -namespace blockstore { - -class Data { -public: - explicit Data(size_t size); - Data(Data &&rhs); // move constructor - virtual ~Data(); - - Data copy() const; - - void *data(); - const void *data() const; - - size_t size() const; - - Data &FillWithZeroes(); - - void StoreToFile(const boost::filesystem::path &filepath) const; - static Data LoadFromFile(const boost::filesystem::path &filepath); - -private: - size_t _size; - void *_data; - - static void _assertFileExists(const std::ifstream &file, const boost::filesystem::path &filepath); - static size_t _getStreamSize(std::istream &stream); - void _readFromStream(std::istream &stream); - - DISALLOW_COPY_AND_ASSIGN(Data); -}; - -//TODO Test operator== -bool operator==(const Data &lhs, const Data &rhs); - -} - -#endif diff --git a/utils/FixedSizeData.h b/utils/FixedSizeData.h deleted file mode 100644 index 1199275f..00000000 --- a/utils/FixedSizeData.h +++ /dev/null @@ -1,112 +0,0 @@ -#pragma once -#ifndef BLOCKSTORE_UTILS_data_H_ -#define BLOCKSTORE_UTILS_data_H_ - -#include -#include -#include -#include - -namespace blockstore { - -template -class FixedSizeData { -public: - //Non-virtual destructor because we want objects to be small - ~FixedSizeData() {} - - static constexpr unsigned int BINARY_LENGTH = SIZE; - static constexpr unsigned int STRING_LENGTH = 2 * BINARY_LENGTH; // Hex encoding - - static FixedSizeData CreateRandom(); - - static FixedSizeData FromString(const std::string &data); - std::string ToString() const; - - static FixedSizeData FromBinary(const void *source); - void ToBinary(void *target) const; - - const unsigned char *data() const; - -private: - FixedSizeData() {} - static CryptoPP::AutoSeededRandomPool &RandomPool(); - - unsigned char _data[BINARY_LENGTH]; -}; - -template bool operator==(const FixedSizeData &lhs, const FixedSizeData &rhs); -template bool operator!=(const FixedSizeData &lhs, const FixedSizeData &rhs); - -// ----- Implementation ----- - -template constexpr unsigned int FixedSizeData::BINARY_LENGTH; -template constexpr unsigned int FixedSizeData::STRING_LENGTH; - -template -CryptoPP::AutoSeededRandomPool &FixedSizeData::RandomPool() { - static CryptoPP::AutoSeededRandomPool singleton; - return singleton; -} - -template -FixedSizeData FixedSizeData::CreateRandom() { - FixedSizeData result; - RandomPool().GenerateBlock(result._data, BINARY_LENGTH); - return result; -} - -template -FixedSizeData FixedSizeData::FromString(const std::string &data) { - assert(data.size() == STRING_LENGTH); - FixedSizeData result; - CryptoPP::StringSource(data, true, - new CryptoPP::HexDecoder( - new CryptoPP::ArraySink(result._data, BINARY_LENGTH) - ) - ); - return result; -} - -template -std::string FixedSizeData::ToString() const { - std::string result; - CryptoPP::ArraySource(_data, BINARY_LENGTH, true, - new CryptoPP::HexEncoder( - new CryptoPP::StringSink(result) - ) - ); - assert(result.size() == STRING_LENGTH); - return result; -} - -template -const unsigned char *FixedSizeData::data() const { - return _data; -} - -template -void FixedSizeData::ToBinary(void *target) const { - std::memcpy(target, _data, BINARY_LENGTH); -} - -template -FixedSizeData FixedSizeData::FromBinary(const void *source) { - FixedSizeData result; - std::memcpy(result._data, source, BINARY_LENGTH); - return result; -} - -template -bool operator==(const FixedSizeData &lhs, const FixedSizeData &rhs) { - return 0 == std::memcmp(lhs.data(), rhs.data(), FixedSizeData::BINARY_LENGTH); -} - -template -bool operator!=(const FixedSizeData &lhs, const FixedSizeData &rhs) { - return !operator==(lhs, rhs); -} - -} - -#endif diff --git a/utils/Key.h b/utils/Key.h index cd06ecda..e0b03f31 100644 --- a/utils/Key.h +++ b/utils/Key.h @@ -3,12 +3,12 @@ #define BLOCKSTORE_UTILS_KEY_H_ #include -#include "FixedSizeData.h" +#include namespace blockstore { // A key here is NOT a key for encryption, but a key as used in key->value mappings ("access handle for a block"). -using Key = FixedSizeData<16>; +using Key = cpputils::FixedSizeData<16>; } From 3e12c771a15834985d56ad9d1ce9fe61e28e513d Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sat, 25 Apr 2015 03:47:03 +0200 Subject: [PATCH 13/41] Remove dependency to tempfile --- biicode.conf | 1 - test/implementations/ondisk/OnDiskBlockStoreTest.cpp | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/biicode.conf b/biicode.conf index 29a2833a..815f499f 100644 --- a/biicode.conf +++ b/biicode.conf @@ -7,7 +7,6 @@ messmer/cmake: 3 messmer/cpp-utils: 2 messmer/parallelaccessstore: 0 - messmer/tempfile: 4 [parent] messmer/blockstore: 1 diff --git a/test/implementations/ondisk/OnDiskBlockStoreTest.cpp b/test/implementations/ondisk/OnDiskBlockStoreTest.cpp index 1ee64b0e..884e5516 100644 --- a/test/implementations/ondisk/OnDiskBlockStoreTest.cpp +++ b/test/implementations/ondisk/OnDiskBlockStoreTest.cpp @@ -4,7 +4,7 @@ #include "../../testutils/BlockStoreWithRandomKeysTest.h" #include "google/gtest/gtest.h" -#include "messmer/tempfile/src/TempDir.h" +#include "messmer/cpp-utils/tempfile/TempDir.h" using blockstore::BlockStore; @@ -14,7 +14,7 @@ using blockstore::ondisk::OnDiskBlockStore; using std::unique_ptr; using std::make_unique; -using tempfile::TempDir; +using cpputils::TempDir; class OnDiskBlockStoreTestFixture: public BlockStoreTestFixture { public: From a2c89ed73a207b30275f706fd54f885031aef4e7 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sat, 25 Apr 2015 16:43:52 +0200 Subject: [PATCH 14/41] Use cpputils::DataFixture instead of DataBlockFixture --- test/implementations/encrypted/CipherTest.cpp | 15 +++---- .../OnDiskBlockTest/OnDiskBlockFlushTest.cpp | 8 ++-- .../OnDiskBlockTest/OnDiskBlockLoadTest.cpp | 15 +++---- .../helpers/BlockStoreWithRandomKeysTest.cpp | 6 +-- test/testutils/BlockStoreTest_Data.h | 7 +-- test/testutils/BlockStoreTest_Size.h | 45 ++++++++----------- test/utils/BlockStoreUtilsTest.cpp | 8 ++-- 7 files changed, 42 insertions(+), 62 deletions(-) diff --git a/test/implementations/encrypted/CipherTest.cpp b/test/implementations/encrypted/CipherTest.cpp index ec84837b..620e60db 100644 --- a/test/implementations/encrypted/CipherTest.cpp +++ b/test/implementations/encrypted/CipherTest.cpp @@ -3,14 +3,14 @@ #include "../../../implementations/encrypted/ciphers/AES256_GCM.h" #include "../../../implementations/encrypted/ciphers/Cipher.h" -#include +#include #include #include using namespace blockstore::encrypted; using cpputils::Data; -using cpputils::DataBlockFixture; +using cpputils::DataFixture; template class CipherTest: public ::testing::Test { @@ -19,7 +19,7 @@ public: typename Cipher::EncryptionKey encKey = createRandomKey(); static typename Cipher::EncryptionKey createRandomKey(int seed = 0) { - DataBlockFixture data(Cipher::EncryptionKey::BINARY_LENGTH, seed); + Data data = DataFixture::generate(Cipher::EncryptionKey::BINARY_LENGTH, seed); return Cipher::EncryptionKey::FromBinary(data.data()); } @@ -50,16 +50,11 @@ public: } static Data CreateZeroes(unsigned int size) { - Data zeroes(size); - zeroes.FillWithZeroes(); - return zeroes; + return std::move(Data(size).FillWithZeroes()); } static Data CreateData(unsigned int size, unsigned int seed = 0) { - DataBlockFixture data(size, seed); - Data result(size); - std::memcpy(result.data(), data.data(), size); - return result; + return DataFixture::generate(size, seed); } }; diff --git a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp index 9ac4c602..2ef0b2a2 100644 --- a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp +++ b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp @@ -1,6 +1,6 @@ #include "../../../../implementations/ondisk/FileAlreadyExistsException.h" #include "../../../../implementations/ondisk/OnDiskBlock.h" -#include +#include #include "google/gtest/gtest.h" #include @@ -12,7 +12,7 @@ using ::testing::Values; using std::unique_ptr; using cpputils::Data; -using cpputils::DataBlockFixture; +using cpputils::DataFixture; using cpputils::TempFile; using cpputils::TempDir; @@ -28,13 +28,13 @@ public: : dir(), key(Key::FromString("1491BB4932A389EE14BC7090AC772972")), file(dir.path() / key.ToString(), false), - randomData(GetParam()) { + randomData(DataFixture::generate(GetParam())) { } TempDir dir; Key key; TempFile file; - DataBlockFixture randomData; + Data randomData; unique_ptr CreateBlockAndLoadItFromDisk() { { diff --git a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp index 364a2033..8fdf8ff6 100644 --- a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp +++ b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockLoadTest.cpp @@ -1,5 +1,5 @@ #include "../../../../implementations/ondisk/OnDiskBlock.h" -#include +#include #include "../../../../utils/FileDoesntExistException.h" #include "google/gtest/gtest.h" @@ -16,7 +16,7 @@ using std::ofstream; using std::unique_ptr; using std::ios; using cpputils::Data; -using cpputils::DataBlockFixture; +using cpputils::DataFixture; using cpputils::TempFile; using cpputils::TempDir; @@ -41,18 +41,15 @@ public: data.StoreToFile(file.path()); } - void StoreData(const DataBlockFixture &data) { - //TODO Implement data.StoreToFile(filepath) instead - Data dataobj(data.size()); - std::memcpy(dataobj.data(), data.data(), data.size()); - dataobj.StoreToFile(file.path()); + void StoreData(const Data &data) { + data.StoreToFile(file.path()); } unique_ptr LoadBlock() { return OnDiskBlock::LoadFromDisk(dir.path(), key); } - void EXPECT_BLOCK_DATA_EQ(const DataBlockFixture &expected, const OnDiskBlock &actual) { + void EXPECT_BLOCK_DATA_EQ(const Data &expected, const OnDiskBlock &actual) { EXPECT_EQ(expected.size(), actual.size()); EXPECT_EQ(0, std::memcmp(expected.data(), actual.data(), expected.size())); } @@ -68,7 +65,7 @@ TEST_P(OnDiskBlockLoadTest, FileSizeIsCorrect) { } TEST_P(OnDiskBlockLoadTest, LoadedDataIsCorrect) { - DataBlockFixture randomData(GetParam()); + Data randomData = DataFixture::generate(GetParam()); StoreData(randomData); auto block = LoadBlock(); diff --git a/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp b/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp index 78a8535b..d4c0513e 100644 --- a/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp +++ b/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp @@ -1,7 +1,7 @@ #include "../../../interface/helpers/BlockStoreWithRandomKeys.h" #include "google/gtest/gtest.h" #include "google/gmock/gmock.h" -#include +#include using ::testing::Test; using ::testing::_; @@ -14,7 +14,7 @@ using std::string; using std::unique_ptr; using std::make_unique; using cpputils::Data; -using cpputils::DataBlockFixture; +using cpputils::DataFixture; using namespace blockstore; @@ -49,7 +49,7 @@ public: const blockstore::Key key = Key::FromString("1491BB4932A389EE14BC7090AC772972"); Data createDataWithSize(size_t size) { - DataBlockFixture fixture(size); + Data fixture(DataFixture::generate(size)); Data data(size); std::memcpy(data.data(), fixture.data(), size); return data; diff --git a/test/testutils/BlockStoreTest_Data.h b/test/testutils/BlockStoreTest_Data.h index 30acbff1..daac33d8 100644 --- a/test/testutils/BlockStoreTest_Data.h +++ b/test/testutils/BlockStoreTest_Data.h @@ -12,11 +12,8 @@ public: BlockStoreDataParametrizedTest(std::unique_ptr blockStore_, const DataRange &testData_) : blockStore(std::move(blockStore_)), testData(testData_), - foregroundData(testData.count), backgroundData(testData.blocksize) { - cpputils::DataBlockFixture _foregroundData(testData.count); - cpputils::DataBlockFixture _backgroundData(testData.blocksize); - std::memcpy(foregroundData.data(), _foregroundData.data(), foregroundData.size()); - std::memcpy(backgroundData.data(), _backgroundData.data(), backgroundData.size()); + foregroundData(cpputils::DataFixture::generate(testData.count, 0)), + backgroundData(cpputils::DataFixture::generate(testData.blocksize, 1)) { } void TestWriteAndReadImmediately() { diff --git a/test/testutils/BlockStoreTest_Size.h b/test/testutils/BlockStoreTest_Size.h index 02679e20..20ee3fbb 100644 --- a/test/testutils/BlockStoreTest_Size.h +++ b/test/testutils/BlockStoreTest_Size.h @@ -1,7 +1,7 @@ // This file is meant to be included by BlockStoreTest.h only #include -#include +#include class BlockStoreSizeParameterizedTest { public: @@ -19,39 +19,34 @@ public: } void TestCreatedBlockData() { - cpputils::DataBlockFixture dataFixture(size); - cpputils::Data data(size); - std::memcpy(data.data(), dataFixture.data(), size); - auto block = blockStore->create(data); - EXPECT_EQ(0, std::memcmp(dataFixture.data(), block->data(), size)); - + cpputils::Data data = cpputils::DataFixture::generate(size); + auto block = blockStore->create(data); + EXPECT_EQ(0, std::memcmp(data.data(), block->data(), size)); } void TestLoadingUnchangedBlockData() { - cpputils::DataBlockFixture dataFixture(size); - cpputils::Data data(size); - std::memcpy(data.data(), dataFixture.data(), size); - blockstore::Key key = blockStore->create(data)->key(); + cpputils::Data data = cpputils::DataFixture::generate(size); + blockstore::Key key = blockStore->create(data)->key(); auto loaded_block = blockStore->load(key); - EXPECT_EQ(0, std::memcmp(dataFixture.data(), loaded_block->data(), size)); + EXPECT_EQ(0, std::memcmp(data.data(), loaded_block->data(), size)); } void TestLoadedBlockIsCorrect() { - cpputils::DataBlockFixture randomData(size); + cpputils::Data randomData = cpputils::DataFixture::generate(size); auto loaded_block = StoreDataToBlockAndLoadIt(randomData); EXPECT_EQ(size, loaded_block->size()); EXPECT_EQ(0, std::memcmp(randomData.data(), loaded_block->data(), size)); } void TestLoadedBlockIsCorrectWhenLoadedDirectlyAfterFlushing() { - cpputils::DataBlockFixture randomData(size); + cpputils::Data randomData = cpputils::DataFixture::generate(size); auto loaded_block = StoreDataToBlockAndLoadItDirectlyAfterFlushing(randomData); EXPECT_EQ(size, loaded_block->size()); EXPECT_EQ(0, std::memcmp(randomData.data(), loaded_block->data(), size)); } void TestAfterCreate_FlushingDoesntChangeBlock() { - cpputils::DataBlockFixture randomData(size); + cpputils::Data randomData = cpputils::DataFixture::generate(size); auto block = CreateBlock(); WriteDataToBlock(block.get(), randomData); block->flush(); @@ -60,7 +55,7 @@ public: } void TestAfterLoad_FlushingDoesntChangeBlock() { - cpputils::DataBlockFixture randomData(size); + cpputils::Data randomData = cpputils::DataFixture::generate(size); auto block = CreateBlockAndLoadIt(); WriteDataToBlock(block.get(), randomData); block->flush(); @@ -69,7 +64,7 @@ public: } void TestAfterCreate_FlushesWhenDestructed() { - cpputils::DataBlockFixture randomData(size); + cpputils::Data randomData = cpputils::DataFixture::generate(size); blockstore::Key key = key; { auto block = blockStore->create(cpputils::Data(size)); @@ -81,7 +76,7 @@ public: } void TestAfterLoad_FlushesWhenDestructed() { - cpputils::DataBlockFixture randomData(size); + cpputils::Data randomData = cpputils::DataFixture::generate(size); blockstore::Key key = key; { key = CreateBlock()->key(); @@ -109,20 +104,16 @@ private: return ZEROES; } - std::unique_ptr StoreDataToBlockAndLoadIt(const cpputils::DataBlockFixture &data) { + std::unique_ptr StoreDataToBlockAndLoadIt(const cpputils::Data &data) { blockstore::Key key = StoreDataToBlockAndGetKey(data); return blockStore->load(key); } - blockstore::Key StoreDataToBlockAndGetKey(const cpputils::DataBlockFixture &dataFixture) { - cpputils::Data data(dataFixture.size()); - std::memcpy(data.data(), dataFixture.data(), dataFixture.size()); + blockstore::Key StoreDataToBlockAndGetKey(const cpputils::Data &data) { return blockStore->create(data)->key(); } - std::unique_ptr StoreDataToBlockAndLoadItDirectlyAfterFlushing(const cpputils::DataBlockFixture &dataFixture) { - cpputils::Data data(dataFixture.size()); - std::memcpy(data.data(), dataFixture.data(), dataFixture.size()); + std::unique_ptr StoreDataToBlockAndLoadItDirectlyAfterFlushing(const cpputils::Data &data) { auto block = blockStore->create(data); block->flush(); return blockStore->load(block->key()); @@ -137,11 +128,11 @@ private: return blockStore->create(cpputils::Data(size)); } - void WriteDataToBlock(blockstore::Block *block, const cpputils::DataBlockFixture &randomData) { + void WriteDataToBlock(blockstore::Block *block, const cpputils::Data &randomData) { block->write(randomData.data(), 0, randomData.size()); } - void EXPECT_BLOCK_DATA_CORRECT(const blockstore::Block &block, const cpputils::DataBlockFixture &randomData) { + void EXPECT_BLOCK_DATA_CORRECT(const blockstore::Block &block, const cpputils::Data &randomData) { EXPECT_EQ(randomData.size(), block.size()); EXPECT_EQ(0, std::memcmp(randomData.data(), block.data(), randomData.size())); } diff --git a/test/utils/BlockStoreUtilsTest.cpp b/test/utils/BlockStoreUtilsTest.cpp index 4d0aca82..adc72ad8 100644 --- a/test/utils/BlockStoreUtilsTest.cpp +++ b/test/utils/BlockStoreUtilsTest.cpp @@ -1,5 +1,5 @@ #include "../../implementations/testfake/FakeBlockStore.h" -#include +#include #include "../../utils/BlockStoreUtils.h" #include "google/gtest/gtest.h" @@ -12,7 +12,7 @@ using ::testing::Values; using std::make_unique; using std::unique_ptr; using cpputils::Data; -using cpputils::DataBlockFixture; +using cpputils::DataFixture; using namespace blockstore; using namespace blockstore::utils; @@ -24,13 +24,13 @@ public: unsigned int SIZE = 1024 * 1024; BlockStoreUtilsTest(): ZEROES(SIZE), - dataFixture(SIZE), + dataFixture(DataFixture::generate(SIZE)), blockStore(make_unique()) { ZEROES.FillWithZeroes(); } Data ZEROES; - DataBlockFixture dataFixture; + Data dataFixture; unique_ptr blockStore; }; From 77b67a8137b3e29f6aefb1e341a1c07ccaea78c8 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sat, 25 Apr 2015 17:17:06 +0200 Subject: [PATCH 15/41] Reduce use of memcmp and use operator==(Data, Data) instead --- test/implementations/encrypted/CipherTest.cpp | 5 ++--- .../ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp | 2 +- .../ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp | 11 +++-------- test/testutils/BlockStoreTest_Data.h | 7 +------ 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/test/implementations/encrypted/CipherTest.cpp b/test/implementations/encrypted/CipherTest.cpp index 620e60db..609a2717 100644 --- a/test/implementations/encrypted/CipherTest.cpp +++ b/test/implementations/encrypted/CipherTest.cpp @@ -26,14 +26,13 @@ public: void CheckEncryptThenDecryptIsIdentity(const Data &plaintext) { Data ciphertext = Encrypt(plaintext); Data decrypted = Decrypt(ciphertext); - EXPECT_EQ(plaintext.size(), decrypted.size()); - EXPECT_EQ(0, std::memcmp(plaintext.data(), decrypted.data(), plaintext.size())); + EXPECT_EQ(plaintext, decrypted); } void CheckEncryptIsIndeterministic(const Data &plaintext) { Data ciphertext = Encrypt(plaintext); Data ciphertext2 = Encrypt(plaintext); - EXPECT_NE(0, std::memcmp(ciphertext.data(), ciphertext2.data(), ciphertext.size())); + EXPECT_NE(ciphertext, ciphertext2); } void CheckEncryptedSize(const Data &plaintext) { diff --git a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp index 0a36ebc4..5f1fe07d 100644 --- a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp +++ b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp @@ -69,7 +69,7 @@ TEST_P(OnDiskBlockCreateSizeTest, OnDiskSizeIsCorrect) { TEST_P(OnDiskBlockCreateSizeTest, OnDiskBlockIsZeroedOut) { Data fileContent = Data::LoadFromFile(file.path()).value(); - EXPECT_EQ(0, std::memcmp(ZEROES.data(), fileContent.data(), fileContent.size())); + EXPECT_EQ(ZEROES, fileContent); } // This test is also tested by OnDiskBlockStoreTest, but there the block is created using the BlockStore interface. diff --git a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp index 2ef0b2a2..f0638639 100644 --- a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp +++ b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp @@ -38,17 +38,13 @@ public: unique_ptr CreateBlockAndLoadItFromDisk() { { - Data data(randomData.size()); - std::memcpy(data.data(), randomData.data(), randomData.size()); - auto block = OnDiskBlock::CreateOnDisk(dir.path(), key, std::move(data)); + OnDiskBlock::CreateOnDisk(dir.path(), key, randomData.copy()); } return OnDiskBlock::LoadFromDisk(dir.path(), key); } unique_ptr CreateBlock() { - Data data(randomData.size()); - std::memcpy(data.data(), randomData.data(), randomData.size()); - return OnDiskBlock::CreateOnDisk(dir.path(), key, std::move(data)); + return OnDiskBlock::CreateOnDisk(dir.path(), key, randomData.copy()); } void WriteDataToBlock(const unique_ptr &block) { @@ -62,8 +58,7 @@ public: void EXPECT_STORED_FILE_DATA_CORRECT() { Data actual = Data::LoadFromFile(file.path()).value(); - EXPECT_EQ(randomData.size(), actual.size()); - EXPECT_EQ(0, std::memcmp(randomData.data(), actual.data(), randomData.size())); + EXPECT_EQ(randomData, actual); } }; INSTANTIATE_TEST_CASE_P(OnDiskBlockFlushTest, OnDiskBlockFlushTest, Values((size_t)0, (size_t)1, (size_t)1024, (size_t)4096, (size_t)10*1024*1024)); diff --git a/test/testutils/BlockStoreTest_Data.h b/test/testutils/BlockStoreTest_Data.h index daac33d8..202c8b32 100644 --- a/test/testutils/BlockStoreTest_Data.h +++ b/test/testutils/BlockStoreTest_Data.h @@ -46,11 +46,6 @@ private: cpputils::Data foregroundData; cpputils::Data backgroundData; - void EXPECT_DATA_EQ(const cpputils::Data &expected, const cpputils::Data &actual) { - EXPECT_EQ(expected.size(), actual.size()); - EXPECT_EQ(0, std::memcmp(expected.data(), actual.data(), expected.size())); - } - blockstore::Key CreateBlockWriteToItAndReturnKey(const cpputils::Data &to_write) { auto newblock = blockStore->create(cpputils::Data(testData.blocksize).FillWithZeroes()); @@ -61,7 +56,7 @@ private: void EXPECT_DATA_READS_AS(const cpputils::Data &expected, const blockstore::Block &block, off_t offset, size_t count) { cpputils::Data read(count); std::memcpy(read.data(), (uint8_t*)block.data() + offset, count); - EXPECT_DATA_EQ(expected, read); + EXPECT_EQ(expected, read); } void EXPECT_DATA_READS_AS_OUTSIDE_OF(const cpputils::Data &expected, const blockstore::Block &block, off_t start, size_t count) { From 1f14598d2570e33412dbe0e435bf8cb21a2a789a Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sun, 26 Apr 2015 12:32:20 +0200 Subject: [PATCH 16/41] Added Test Cases for PeriodicTask --- implementations/caching/PeriodicTask.cpp | 15 ++--- implementations/caching/PeriodicTask.h | 1 - .../caching/PeriodicTaskTest.cpp | 63 +++++++++++++++++++ 3 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 test/implementations/caching/PeriodicTaskTest.cpp diff --git a/implementations/caching/PeriodicTask.cpp b/implementations/caching/PeriodicTask.cpp index 1b260530..273bb15e 100644 --- a/implementations/caching/PeriodicTask.cpp +++ b/implementations/caching/PeriodicTask.cpp @@ -9,17 +9,18 @@ namespace caching { PeriodicTask::PeriodicTask(function task, double intervalSec) : _thread(), _task(task), _intervalSec(intervalSec) { _thread = boost::thread([this]() { - boost::chrono::nanoseconds interval((uint64_t)(UINT64_C(1000000000) * _intervalSec)); - try { + boost::chrono::nanoseconds interval((uint64_t)(UINT64_C(1000000000) * _intervalSec)); + try { while(true) { boost::this_thread::sleep_for(interval); _task(); } - } catch (const boost::thread_interrupted &e) { - //Do nothing, exit thread. - } catch (...) { - cerr << "PeriodicTask crashed" << endl; - } + } catch (const boost::thread_interrupted &e) { + //Do nothing, exit thread. + } catch (...) { + //TODO Think about logging + cerr << "PeriodicTask crashed" << endl; + } }); } diff --git a/implementations/caching/PeriodicTask.h b/implementations/caching/PeriodicTask.h index 4ad221e2..6a0955e6 100644 --- a/implementations/caching/PeriodicTask.h +++ b/implementations/caching/PeriodicTask.h @@ -7,7 +7,6 @@ namespace blockstore { namespace caching { -//TODO Test cases class PeriodicTask { public: PeriodicTask(std::function task, double intervalSec); diff --git a/test/implementations/caching/PeriodicTaskTest.cpp b/test/implementations/caching/PeriodicTaskTest.cpp new file mode 100644 index 00000000..4e945961 --- /dev/null +++ b/test/implementations/caching/PeriodicTaskTest.cpp @@ -0,0 +1,63 @@ +#include + +#include "../../../implementations/caching/PeriodicTask.h" + +#include +#include +#include + +using ::testing::Test; +using std::mutex; +using std::unique_lock; +using std::condition_variable; + +using namespace blockstore::caching; + +class AtomicCounter { +public: + AtomicCounter(int count): _mutex(), _cv(), _counter(count) {} + + void decrease() { + unique_lock lock(_mutex); + --_counter; + _cv.notify_all(); + } + + void waitForZero() { + unique_lock lock(_mutex); + _cv.wait(lock, [this] () {return _counter <= 0;}); + } +private: + mutex _mutex; + condition_variable _cv; + int _counter; +}; + +class PeriodicTaskTest: public Test { +}; + +TEST_F(PeriodicTaskTest, DoesntDeadlockInDestructorWhenDestructedImmediately) { + PeriodicTask task([](){}, 1); +} + +TEST_F(PeriodicTaskTest, CallsCallbackAtLeast10Times) { + AtomicCounter counter(10); + + PeriodicTask task([&counter](){ + counter.decrease(); + }, 0.001); + + counter.waitForZero(); +} + +TEST_F(PeriodicTaskTest, DoesntCallCallbackAfterDestruction) { + std::atomic callCount(0); + { + PeriodicTask task([&callCount](){ + callCount += 1; + }, 0.001); + } + int callCountDirectlyAfterDestruction = callCount; + boost::this_thread::sleep_for(boost::chrono::seconds(1)); + EXPECT_EQ(callCountDirectlyAfterDestruction, callCount); +} From 09bc28e81064371f4d09946864242674a073e5ee Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sun, 26 Apr 2015 16:36:48 +0200 Subject: [PATCH 17/41] EncryptedBlock stores its block key in the block, so that an attacker can't replace block contents with a different block --- implementations/encrypted/EncryptedBlock.h | 60 +++++++++++++------ .../encrypted/EncryptedBlockStoreTest.cpp | 5 +- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/implementations/encrypted/EncryptedBlock.h b/implementations/encrypted/EncryptedBlock.h index 2b17e843..d648c76b 100644 --- a/implementations/encrypted/EncryptedBlock.h +++ b/implementations/encrypted/EncryptedBlock.h @@ -15,8 +15,6 @@ namespace blockstore { namespace encrypted { template class EncryptedBlockStore; -//TODO not only encryption, but also hmac - template class EncryptedBlock: public Block { public: @@ -25,11 +23,11 @@ public: static std::unique_ptr TryDecrypt(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key); //TODO Storing key twice (in parent class and in object pointed to). Once would be enough. - EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key, cpputils::Data plaintextData); + EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &key, cpputils::Data plaintextWithHeader); virtual ~EncryptedBlock(); const void *data() const override; - void write(const void *source, uint64_t offset, uint64_t size) override; + void write(const void *source, uint64_t offset, uint64_t count) override; void flush() override; size_t size() const override; @@ -38,44 +36,70 @@ public: private: std::unique_ptr _baseBlock; - cpputils::Data _plaintextData; + cpputils::Data _plaintextWithHeader; typename Cipher::EncryptionKey _encKey; bool _dataChanged; + static constexpr unsigned int HEADER_LENGTH = Key::BINARY_LENGTH; + void _encryptToBaseBlock(); + static cpputils::Data _prependKeyHeaderToData(const Key &key, cpputils::Data data); + static bool _keyHeaderIsCorrect(const Key &key, const cpputils::Data &data); DISALLOW_COPY_AND_ASSIGN(EncryptedBlock); }; +template +constexpr unsigned int EncryptedBlock::HEADER_LENGTH; template std::unique_ptr> EncryptedBlock::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey) { - cpputils::Data encrypted = Cipher::encrypt((byte*)data.data(), data.size(), encKey); + cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, std::move(data)); + cpputils::Data encrypted = Cipher::encrypt((byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), encKey); auto baseBlock = baseBlockStore->tryCreate(key, std::move(encrypted)); if (baseBlock.get() == nullptr) { //TODO Test this code branch return nullptr; } - return std::make_unique(std::move(baseBlock), encKey, std::move(data)); + return std::make_unique(std::move(baseBlock), encKey, std::move(plaintextWithHeader)); } template std::unique_ptr> EncryptedBlock::TryDecrypt(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &encKey) { //TODO Change BlockStore so we can read their "class Data" objects instead of "void *data()", and then we can change the Cipher interface to take Data objects instead of "byte *" + size - boost::optional plaintext = Cipher::decrypt((byte*)baseBlock->data(), baseBlock->size(), encKey); - if(!plaintext) { + boost::optional plaintextWithHeader = Cipher::decrypt((byte*)baseBlock->data(), baseBlock->size(), encKey); + if(!plaintextWithHeader) { + //Decryption failed (e.g. an authenticated cipher detected modifications to the ciphertext) return nullptr; } - return std::make_unique>(std::move(baseBlock), encKey, std::move(*plaintext)); + if(!_keyHeaderIsCorrect(baseBlock->key(), *plaintextWithHeader)) { + //The stored key in the block data is incorrect - an attacker might have exchanged the contents with the encrypted data from a different block + return nullptr; + } + return std::make_unique>(std::move(baseBlock), encKey, std::move(*plaintextWithHeader)); } template -EncryptedBlock::EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &encKey, cpputils::Data plaintextData) +cpputils::Data EncryptedBlock::_prependKeyHeaderToData(const Key &key, cpputils::Data data) { + static_assert(HEADER_LENGTH >= Key::BINARY_LENGTH, "Key doesn't fit into the header"); + cpputils::Data result(data.size() + HEADER_LENGTH); + std::memcpy(result.data(), key.data(), Key::BINARY_LENGTH); + std::memcpy((uint8_t*)result.data() + Key::BINARY_LENGTH, data.data(), data.size()); + return result; +} + +template +bool EncryptedBlock::_keyHeaderIsCorrect(const Key &key, const cpputils::Data &data) { + return 0 == std::memcmp(key.data(), data.data(), Key::BINARY_LENGTH); +} + +template +EncryptedBlock::EncryptedBlock(std::unique_ptr baseBlock, const typename Cipher::EncryptionKey &encKey, cpputils::Data plaintextWithHeader) :Block(baseBlock->key()), _baseBlock(std::move(baseBlock)), - _plaintextData(std::move(plaintextData)), + _plaintextWithHeader(std::move(plaintextWithHeader)), _encKey(encKey), _dataChanged(false) { } @@ -87,13 +111,13 @@ EncryptedBlock::~EncryptedBlock() { template const void *EncryptedBlock::data() const { - return _plaintextData.data(); + return (uint8_t*)_plaintextWithHeader.data() + HEADER_LENGTH; } template -void EncryptedBlock::write(const void *source, uint64_t offset, uint64_t size) { - assert(offset <= _plaintextData.size() && offset + size <= _plaintextData.size()); //Also check offset < _data->size() because of possible overflow in the addition - std::memcpy((uint8_t*)_plaintextData.data()+offset, source, size); +void EncryptedBlock::write(const void *source, uint64_t offset, uint64_t count) { + assert(offset <= size() && offset + count <= size()); //Also check offset < size() because of possible overflow in the addition + std::memcpy((uint8_t*)_plaintextWithHeader.data()+HEADER_LENGTH+offset, source, count); _dataChanged = true; } @@ -105,13 +129,13 @@ void EncryptedBlock::flush() { template size_t EncryptedBlock::size() const { - return _plaintextData.size(); + return _plaintextWithHeader.size() - HEADER_LENGTH; } template void EncryptedBlock::_encryptToBaseBlock() { if (_dataChanged) { - cpputils::Data encrypted = Cipher::encrypt((byte*)_plaintextData.data(), _plaintextData.size(), _encKey); + cpputils::Data encrypted = Cipher::encrypt((byte*)_plaintextWithHeader.data(), _plaintextWithHeader.size(), _encKey); _baseBlock->write(encrypted.data(), 0, encrypted.size()); _dataChanged = false; } diff --git a/test/implementations/encrypted/EncryptedBlockStoreTest.cpp b/test/implementations/encrypted/EncryptedBlockStoreTest.cpp index 4a8b35bb..dd5479ac 100644 --- a/test/implementations/encrypted/EncryptedBlockStoreTest.cpp +++ b/test/implementations/encrypted/EncryptedBlockStoreTest.cpp @@ -22,4 +22,7 @@ public: INSTANTIATE_TYPED_TEST_CASE_P(Encrypted, BlockStoreTest, EncryptedBlockStoreTestFixture); -//TODO Add specific tests, for example loading it with a different key doesn't work +//TODO Add specific tests, for example +// - loading it with a different encKey doesn't work +// - loading it with a different blockstore::Key will fail (because it stores its key in a header) +// - when using an authenticated cipher, loading a modified block will fail From 5f83c133b0d674c8d0412e9bc922604267aed9b8 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sun, 26 Apr 2015 16:45:00 +0200 Subject: [PATCH 18/41] Added error messages when block decryption fails --- implementations/encrypted/EncryptedBlock.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/implementations/encrypted/EncryptedBlock.h b/implementations/encrypted/EncryptedBlock.h index d648c76b..f118875c 100644 --- a/implementations/encrypted/EncryptedBlock.h +++ b/implementations/encrypted/EncryptedBlock.h @@ -72,10 +72,14 @@ std::unique_ptr> EncryptedBlock::TryDecrypt(std:: boost::optional plaintextWithHeader = Cipher::decrypt((byte*)baseBlock->data(), baseBlock->size(), encKey); if(!plaintextWithHeader) { //Decryption failed (e.g. an authenticated cipher detected modifications to the ciphertext) + //TODO Think about logging + std::cerr << "Decrypting block " << baseBlock->key() << " failed. Was the block modified by an attacker?" << std::endl; return nullptr; } if(!_keyHeaderIsCorrect(baseBlock->key(), *plaintextWithHeader)) { //The stored key in the block data is incorrect - an attacker might have exchanged the contents with the encrypted data from a different block + //TODO Think about logging + std::cerr << "Decrypting block " << baseBlock->key() << " failed due to invalid block key. Was the block modified by an attacker?" << std::endl; return nullptr; } return std::make_unique>(std::move(baseBlock), encKey, std::move(*plaintextWithHeader)); From b60b1199856604dfc0018aa614b4c0aeea816ea5 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sun, 26 Apr 2015 22:20:04 +0200 Subject: [PATCH 19/41] Fix iostream use --- implementations/encrypted/EncryptedBlock.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/implementations/encrypted/EncryptedBlock.h b/implementations/encrypted/EncryptedBlock.h index f118875c..3770786e 100644 --- a/implementations/encrypted/EncryptedBlock.h +++ b/implementations/encrypted/EncryptedBlock.h @@ -8,6 +8,7 @@ #include "messmer/cpp-utils/macros.h" #include +#include #include #include "ciphers/Cipher.h" @@ -73,13 +74,13 @@ std::unique_ptr> EncryptedBlock::TryDecrypt(std:: if(!plaintextWithHeader) { //Decryption failed (e.g. an authenticated cipher detected modifications to the ciphertext) //TODO Think about logging - std::cerr << "Decrypting block " << baseBlock->key() << " failed. Was the block modified by an attacker?" << std::endl; + std::cerr << "Decrypting block " << baseBlock->key().ToString() << " failed. Was the block modified by an attacker?" << std::endl; return nullptr; } if(!_keyHeaderIsCorrect(baseBlock->key(), *plaintextWithHeader)) { //The stored key in the block data is incorrect - an attacker might have exchanged the contents with the encrypted data from a different block //TODO Think about logging - std::cerr << "Decrypting block " << baseBlock->key() << " failed due to invalid block key. Was the block modified by an attacker?" << std::endl; + std::cerr << "Decrypting block " << baseBlock->key().ToString() << " failed due to invalid block key. Was the block modified by an attacker?" << std::endl; return nullptr; } return std::make_unique>(std::move(baseBlock), encKey, std::move(*plaintextWithHeader)); From e177c6f45c3a880e7c7c729c836068c5c50d2e78 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 27 Apr 2015 01:22:39 +0200 Subject: [PATCH 20/41] Switch to a QueueMap implementation with less indirections (directly store elements instead of pointers) --- implementations/caching/Cache.cpp | 4 +- implementations/caching/Cache.h | 1 - implementations/caching/QueueMap.h | 79 ++++++---- test/implementations/caching/QueueMapTest.cpp | 137 ++++++++++++++++++ 4 files changed, 190 insertions(+), 31 deletions(-) create mode 100644 test/implementations/caching/QueueMapTest.cpp diff --git a/implementations/caching/Cache.cpp b/implementations/caching/Cache.cpp index d3a9dba1..050dbc40 100644 --- a/implementations/caching/Cache.cpp +++ b/implementations/caching/Cache.cpp @@ -27,7 +27,7 @@ Cache::~Cache() { unique_ptr Cache::pop(const Key &key) { lock_guard lock(_mutex); auto found = _cachedBlocks.pop(key); - if (found.get() == nullptr) { + if (!found) { return nullptr; } auto block = found->releaseBlock(); @@ -42,7 +42,7 @@ void Cache::push(unique_ptr block) { assert(_cachedBlocks.size() == MAX_ENTRIES-1); } Key key = block->key(); - _cachedBlocks.push(key, make_unique(std::move(block))); + _cachedBlocks.push(key, std::move(block)); } void Cache::_popOldEntries() { diff --git a/implementations/caching/Cache.h b/implementations/caching/Cache.h index 740e9003..8ff93ae6 100644 --- a/implementations/caching/Cache.h +++ b/implementations/caching/Cache.h @@ -31,7 +31,6 @@ public: private: void _popOldEntries(); - mutable std::mutex _mutex; QueueMap _cachedBlocks; std::unique_ptr _timeoutFlusher; diff --git a/implementations/caching/QueueMap.h b/implementations/caching/QueueMap.h index 758b2bd4..d21b31cf 100644 --- a/implementations/caching/QueueMap.h +++ b/implementations/caching/QueueMap.h @@ -5,6 +5,8 @@ #include #include #include +#include +#include namespace blockstore { namespace caching { @@ -14,35 +16,43 @@ namespace caching { template class QueueMap { public: - QueueMap(): _entries(), _sentinel(nullptr, nullptr, &_sentinel, &_sentinel) { + QueueMap(): _entries(), _sentinel(&_sentinel, &_sentinel) { } - virtual ~QueueMap() {} - - void push(const Key &key, std::unique_ptr value) { - auto newEntry = std::make_unique(&key, std::move(value), _sentinel.prev, &_sentinel); - _sentinel.prev->next = newEntry.get(); - _sentinel.prev = newEntry.get(); - auto insertResult = _entries.emplace(key, std::move(newEntry)); - assert(insertResult.second == true); + virtual ~QueueMap() { + for (auto &entry : _entries) { + entry.second.release(); + } } - std::unique_ptr pop(const Key &key) { + void push(const Key &key, Value value) { + auto newEntry = _entries.emplace(std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple(_sentinel.prev, &_sentinel)); + assert(newEntry.second == true); + newEntry.first->second.init(&newEntry.first->first, std::move(value)); + //The following is ok, because std::unordered_map never invalidates pointers to its entries + _sentinel.prev->next = &newEntry.first->second; + _sentinel.prev = &newEntry.first->second; + } + + boost::optional pop(const Key &key) { auto found = _entries.find(key); if (found == _entries.end()) { - return nullptr; + return boost::none; } - _removeFromQueue(found->second.get()); - auto value = std::move(found->second->value); + _removeFromQueue(found->second); + auto value = found->second.release(); _entries.erase(found); - return value; + return std::move(value); } - std::unique_ptr pop() { + boost::optional pop() { + if(_sentinel.next == &_sentinel) { + return boost::none; + } return pop(*_sentinel.next->key); } const Value &peek() { - return *_sentinel.next->value; + return _sentinel.next->value(); } uint32_t size() { @@ -50,26 +60,39 @@ public: } private: - struct Entry { - Entry(const Key *key_, std::unique_ptr value_, Entry *prev_, Entry *next_): key(nullptr), value(std::move(value_)), prev(prev_), next(next_) { - if (key_ != nullptr) { - key = std::make_unique(*key_); - } + class Entry { + public: + Entry(Entry *prev_, Entry *next_): prev(prev_), next(next_), key(nullptr), _value() { + } + void init(const Key *key_, Value value_) { + key = key_; + new(_value) Value(std::move(value_)); + } + Value release() { + Value value = std::move(*reinterpret_cast(_value)); + reinterpret_cast(_value)->~Value(); + return value; + } + const Value &value() { + return *reinterpret_cast(_value); } - std::unique_ptr key; - std::unique_ptr value; Entry *prev; Entry *next; + const Key *key; + private: + alignas(Value) char _value[sizeof(Value)]; + DISALLOW_COPY_AND_ASSIGN(Entry); }; - void _removeFromQueue(Entry *entry) { - entry->prev->next = entry->next; - entry->next->prev = entry->prev; + void _removeFromQueue(const Entry &entry) { + entry.prev->next = entry.next; + entry.next->prev = entry.prev; } - //TODO Double indirection unique_ptr and Entry has unique_ptr. Necessary? - std::unordered_map> _entries; + std::unordered_map _entries; Entry _sentinel; + + DISALLOW_COPY_AND_ASSIGN(QueueMap); }; } diff --git a/test/implementations/caching/QueueMapTest.cpp b/test/implementations/caching/QueueMapTest.cpp new file mode 100644 index 00000000..90447d0a --- /dev/null +++ b/test/implementations/caching/QueueMapTest.cpp @@ -0,0 +1,137 @@ +#include +#include +#include "../../../implementations/caching/QueueMap.h" + +using ::testing::Test; + +using namespace blockstore::caching; + +// This is a not-default-constructible Key type +class MinimalKeyType { +public: + static MinimalKeyType create() { + return MinimalKeyType(); + } + bool operator==(const MinimalKeyType &rhs) const { + return true; + } +private: + MinimalKeyType() { + } +}; +namespace std { +template <> struct hash { + size_t operator()(const MinimalKeyType &obj) const { + return 0; + } +}; +} +// This is a not-default-constructible non-copyable but moveable Value type +class MinimalValueType { +public: + static MinimalValueType create() { + return MinimalValueType(); + } + MinimalValueType(MinimalValueType &&rhs) = default; +private: + MinimalValueType() { + } + DISALLOW_COPY_AND_ASSIGN(MinimalValueType); +}; + +class QueueMapTest: public Test { +public: + QueueMap map; +}; + +TEST_F(QueueMapTest, TypeConstraints) { + QueueMap obj; + //Call all functions to ensure they still work + obj.push(MinimalKeyType::create(), MinimalValueType::create()); + obj.peek(); + obj.pop(MinimalKeyType::create()); + obj.push(MinimalKeyType::create(), MinimalValueType::create()); + obj.pop(); + obj.size(); +} + +TEST_F(QueueMapTest, Size_Empty) { + EXPECT_EQ(0, map.size()); +} + +TEST_F(QueueMapTest, Size_AfterPushingOne) { + map.push(2, 3); + EXPECT_EQ(1, map.size()); +} + +TEST_F(QueueMapTest, Size_AfterPushingTwo) { + map.push(2, 3); + map.push(3, 4); + EXPECT_EQ(2, map.size()); +} + +TEST_F(QueueMapTest, Size_AfterPushingTwoAndPoppingOldest) { + map.push(2, 3); + map.push(3, 4); + map.pop(); + EXPECT_EQ(1, map.size()); +} + +TEST_F(QueueMapTest, Size_AfterPushingTwoAndPoppingFirst) { + map.push(2, 3); + map.push(3, 4); + map.pop(2); + EXPECT_EQ(1, map.size()); +} + +TEST_F(QueueMapTest, Size_AfterPushingTwoAndPoppingLast) { + map.push(2, 3); + map.push(3, 4); + map.pop(3); + EXPECT_EQ(1, map.size()); +} + +TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOne) { + map.push(2, 3); + map.pop(); + EXPECT_EQ(0, map.size()); +} + +TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOnePerKey) { + map.push(2, 3); + map.pop(2); + EXPECT_EQ(0, map.size()); +} + +TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOnePushingOne) { + map.push(2, 3); + map.pop(); + map.push(3, 4); + EXPECT_EQ(1, map.size()); +} + +TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOnePerKeyPushingOne) { + map.push(2, 3); + map.pop(2); + map.push(3, 4); + EXPECT_EQ(1, map.size()); +} + +TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOnePushingSame) { + map.push(2, 3); + map.pop(); + map.push(2, 3); + EXPECT_EQ(1, map.size()); +} + +TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOnePerKeyPushingSame) { + map.push(2, 3); + map.pop(2); + map.push(2, 3); + EXPECT_EQ(1, map.size()); +} + +//TODO Pushing the same key twice +//TODO Popping from empty +//TODO Popping invalid key +//TODO Test that in all cases, destructors of Value are called correctly in QueueMap when [a] pop() [b] pop(key) [c] ~QueueMap() From f6473422280df18da051151ed38acdb938c5f796 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 27 Apr 2015 01:36:32 +0200 Subject: [PATCH 21/41] TODOs --- test/implementations/caching/QueueMapTest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/implementations/caching/QueueMapTest.cpp b/test/implementations/caching/QueueMapTest.cpp index 90447d0a..37dc300a 100644 --- a/test/implementations/caching/QueueMapTest.cpp +++ b/test/implementations/caching/QueueMapTest.cpp @@ -135,3 +135,5 @@ TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOnePerKeyPushingSame) { //TODO Popping from empty //TODO Popping invalid key //TODO Test that in all cases, destructors of Value are called correctly in QueueMap when [a] pop() [b] pop(key) [c] ~QueueMap() +//TODO Test that pushing and popping a copy-and-move-constructible object only called 1 copy constructor +//TODO Test that pushing and popping doesn't invalidate objects (e.g, calls too many destructors) From 6f1c39fd21f5a374a2a1c41b72c8da4e1a2701d4 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 27 Apr 2015 10:35:01 +0200 Subject: [PATCH 22/41] QueueMap: Added more test cases and improved interface --- implementations/caching/Cache.cpp | 2 +- implementations/caching/QueueMap.h | 5 +- test/implementations/caching/QueueMapTest.cpp | 500 +++++++++++++++--- 3 files changed, 429 insertions(+), 78 deletions(-) diff --git a/implementations/caching/Cache.cpp b/implementations/caching/Cache.cpp index 050dbc40..e4cbac53 100644 --- a/implementations/caching/Cache.cpp +++ b/implementations/caching/Cache.cpp @@ -47,7 +47,7 @@ void Cache::push(unique_ptr block) { void Cache::_popOldEntries() { lock_guard lock(_mutex); - while(_cachedBlocks.size() > 0 && _cachedBlocks.peek().ageSeconds() > PURGE_LIFETIME_SEC) { + while(_cachedBlocks.size() > 0 && _cachedBlocks.peek()->ageSeconds() > PURGE_LIFETIME_SEC) { _cachedBlocks.pop(); } } diff --git a/implementations/caching/QueueMap.h b/implementations/caching/QueueMap.h index d21b31cf..c8814691 100644 --- a/implementations/caching/QueueMap.h +++ b/implementations/caching/QueueMap.h @@ -51,7 +51,10 @@ public: return pop(*_sentinel.next->key); } - const Value &peek() { + boost::optional peek() { + if(_sentinel.next == &_sentinel) { + return boost::none; + } return _sentinel.next->value(); } diff --git a/test/implementations/caching/QueueMapTest.cpp b/test/implementations/caching/QueueMapTest.cpp index 37dc300a..851180e9 100644 --- a/test/implementations/caching/QueueMapTest.cpp +++ b/test/implementations/caching/QueueMapTest.cpp @@ -1,139 +1,487 @@ #include #include #include "../../../implementations/caching/QueueMap.h" +#include +#include using ::testing::Test; +using std::unique_ptr; +using std::make_unique; using namespace blockstore::caching; // This is a not-default-constructible Key type class MinimalKeyType { public: - static MinimalKeyType create() { - return MinimalKeyType(); + static int instances; + static MinimalKeyType create(int value) { + return MinimalKeyType(value); } bool operator==(const MinimalKeyType &rhs) const { - return true; + return _value == rhs._value; + } + int value() const { + return _value; + } + MinimalKeyType(const MinimalKeyType &rhs): MinimalKeyType(rhs.value()) { + } + ~MinimalKeyType() { + --instances; } private: - MinimalKeyType() { + MinimalKeyType(int value): _value(value) { + ++instances; } + int _value; }; +int MinimalKeyType::instances = 0; namespace std { template <> struct hash { size_t operator()(const MinimalKeyType &obj) const { - return 0; + return obj.value(); } }; } // This is a not-default-constructible non-copyable but moveable Value type class MinimalValueType { public: - static MinimalValueType create() { - return MinimalValueType(); + static int instances; + static MinimalValueType create(int value) { + return MinimalValueType(value); + } + MinimalValueType(MinimalValueType &&rhs): MinimalValueType(rhs.value()) { + rhs._isMoved = true; + } + ~MinimalValueType() { + assert(!_isDestructed); + --instances; + _isDestructed = true; + } + int value() const { + assert(!_isMoved && !_isDestructed); + return _value; } - MinimalValueType(MinimalValueType &&rhs) = default; private: - MinimalValueType() { + MinimalValueType(int value): _value(value), _isMoved(false), _isDestructed(false) { + ++instances; } + int _value; + bool _isMoved; + bool _isDestructed; DISALLOW_COPY_AND_ASSIGN(MinimalValueType); }; +int MinimalValueType::instances = 0; class QueueMapTest: public Test { public: - QueueMap map; + QueueMapTest() { + MinimalKeyType::instances = 0; + MinimalValueType::instances = 0; + _map = make_unique>(); + } + ~QueueMapTest() { + _map.reset(); + EXPECT_EQ(0, MinimalKeyType::instances); + EXPECT_EQ(0, MinimalValueType::instances); + } + void push(int key, int value) { + _map->push(MinimalKeyType::create(key), MinimalValueType::create(value)); + } + boost::optional pop() { + auto elem = _map->pop(); + if (!elem) { + return boost::none; + } + return elem.value().value(); + } + boost::optional pop(int key) { + auto elem = _map->pop(MinimalKeyType::create(key)); + if (!elem) { + return boost::none; + } + return elem.value().value(); + } + boost::optional peek() { + auto elem = _map->peek(); + if (!elem) { + return boost::none; + } + return elem.value().value(); + } + int size() { + return _map->size(); + } +private: + unique_ptr> _map; }; -TEST_F(QueueMapTest, TypeConstraints) { - QueueMap obj; - //Call all functions to ensure they still work - obj.push(MinimalKeyType::create(), MinimalValueType::create()); - obj.peek(); - obj.pop(MinimalKeyType::create()); - obj.push(MinimalKeyType::create(), MinimalValueType::create()); - obj.pop(); - obj.size(); +class QueueMapSizeTest: public QueueMapTest {}; + +TEST_F(QueueMapSizeTest, Empty) { + EXPECT_EQ(0, size()); } -TEST_F(QueueMapTest, Size_Empty) { - EXPECT_EQ(0, map.size()); +TEST_F(QueueMapSizeTest, AfterPushingOne) { + push(2, 3); + EXPECT_EQ(1, size()); } -TEST_F(QueueMapTest, Size_AfterPushingOne) { - map.push(2, 3); - EXPECT_EQ(1, map.size()); +TEST_F(QueueMapSizeTest, AfterPushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_EQ(2, size()); } -TEST_F(QueueMapTest, Size_AfterPushingTwo) { - map.push(2, 3); - map.push(3, 4); - EXPECT_EQ(2, map.size()); +TEST_F(QueueMapSizeTest, AfterPushingTwoAndPoppingOldest) { + push(2, 3); + push(3, 4); + pop(); + EXPECT_EQ(1, size()); } -TEST_F(QueueMapTest, Size_AfterPushingTwoAndPoppingOldest) { - map.push(2, 3); - map.push(3, 4); - map.pop(); - EXPECT_EQ(1, map.size()); +TEST_F(QueueMapSizeTest, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_EQ(1, size()); } -TEST_F(QueueMapTest, Size_AfterPushingTwoAndPoppingFirst) { - map.push(2, 3); - map.push(3, 4); - map.pop(2); - EXPECT_EQ(1, map.size()); +TEST_F(QueueMapSizeTest, AfterPushingTwoAndPoppingLast) { + push(2, 3); + push(3, 4); + pop(3); + EXPECT_EQ(1, size()); } -TEST_F(QueueMapTest, Size_AfterPushingTwoAndPoppingLast) { - map.push(2, 3); - map.push(3, 4); - map.pop(3); - EXPECT_EQ(1, map.size()); +TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOne) { + push(2, 3); + pop(); + EXPECT_EQ(0, size()); } -TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOne) { - map.push(2, 3); - map.pop(); - EXPECT_EQ(0, map.size()); +TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOnePerKey) { + push(2, 3); + pop(2); + EXPECT_EQ(0, size()); } -TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOnePerKey) { - map.push(2, 3); - map.pop(2); - EXPECT_EQ(0, map.size()); +TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOnePushingOne) { + push(2, 3); + pop(); + push(3, 4); + EXPECT_EQ(1, size()); } -TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOnePushingOne) { - map.push(2, 3); - map.pop(); - map.push(3, 4); - EXPECT_EQ(1, map.size()); +TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOnePerKeyPushingOne) { + push(2, 3); + pop(2); + push(3, 4); + EXPECT_EQ(1, size()); } -TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOnePerKeyPushingOne) { - map.push(2, 3); - map.pop(2); - map.push(3, 4); - EXPECT_EQ(1, map.size()); +TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOnePushingSame) { + push(2, 3); + pop(); + push(2, 3); + EXPECT_EQ(1, size()); } -TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOnePushingSame) { - map.push(2, 3); - map.pop(); - map.push(2, 3); - EXPECT_EQ(1, map.size()); +TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOnePerKeyPushingSame) { + push(2, 3); + pop(2); + push(2, 3); + EXPECT_EQ(1, size()); } -TEST_F(QueueMapTest, Size_AfterPushingOnePoppingOnePerKeyPushingSame) { - map.push(2, 3); - map.pop(2); - map.push(2, 3); - EXPECT_EQ(1, map.size()); +class QueueMapMemoryLeakTest: public QueueMapTest { +public: + void EXPECT_NUM_INSTANCES(int num) { + EXPECT_EQ(num, MinimalKeyType::instances); + EXPECT_EQ(num, MinimalValueType::instances); + } +}; + +TEST_F(QueueMapMemoryLeakTest, Empty) { + EXPECT_NUM_INSTANCES(0); +} + +TEST_F(QueueMapMemoryLeakTest, AfterPushingOne) { + push(2, 3); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapMemoryLeakTest, AfterPushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_NUM_INSTANCES(2); +} + +TEST_F(QueueMapMemoryLeakTest, AfterPushingTwoAndPoppingOldest) { + push(2, 3); + push(3, 4); + pop(); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapMemoryLeakTest, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapMemoryLeakTest, AfterPushingTwoAndPoppingLast) { + push(2, 3); + push(3, 4); + pop(3); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOne) { + push(2, 3); + pop(); + EXPECT_NUM_INSTANCES(0); +} + +TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOnePerKey) { + push(2, 3); + pop(2); + EXPECT_NUM_INSTANCES(0); +} + +TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOnePushingOne) { + push(2, 3); + pop(); + push(3, 4); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOnePerKeyPushingOne) { + push(2, 3); + pop(2); + push(3, 4); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOnePushingSame) { + push(2, 3); + pop(); + push(2, 3); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOnePerKeyPushingSame) { + push(2, 3); + pop(2); + push(2, 3); + EXPECT_NUM_INSTANCES(1); +} + +class QueueMapValueTest: public QueueMapTest {}; + +TEST_F(QueueMapValueTest, PoppingFromEmpty) { + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapValueTest, PoppingFromEmptyPerKey) { + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapValueTest, PoppingNonexistingPerKey) { + push(3, 2); + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapValueTest, PushingOne) { + push(3, 2); + EXPECT_EQ(2, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapValueTest, PushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_EQ(3, pop().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapValueTest, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapValueTest, AfterPushingTwoAndPoppingLast) { + push(2, 3); + push(3, 4); + pop(3); + EXPECT_EQ(boost::none, pop(3)); + EXPECT_EQ(3, pop(2).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapValueTest, AfterPushingOnePoppingOne) { + push(2, 3); + pop(); + EXPECT_EQ(boost::none, pop()); + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapValueTest, AfterPushingOnePoppingOnePerKey) { + push(2, 3); + pop(2); + EXPECT_EQ(boost::none, pop()); + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapValueTest, AfterPushingOnePoppingOnePushingOne) { + push(2, 3); + pop(); + push(3, 4); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapValueTest, AfterPushingOnePoppingOnePerKeyPushingOne) { + push(2, 3); + pop(2); + push(3, 4); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapValueTest, PushingSomePoppingMiddlePerKey) { + push(1, 2); + push(2, 3); + push(3, 4); + push(4, 5); + push(5, 6); + EXPECT_EQ(3, pop(2).value()); + EXPECT_EQ(5, pop(4).value()); + EXPECT_EQ(2, pop().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(6, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapValueTest, PushingSomePoppingFirstPerKey) { + push(1, 2); + push(2, 3); + push(3, 4); + push(4, 5); + push(5, 6); + EXPECT_EQ(2, pop(1).value()); + EXPECT_EQ(3, pop(2).value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(5, pop().value()); + EXPECT_EQ(6, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapValueTest, PushingSomePoppingLastPerKey) { + push(1, 2); + push(2, 3); + push(3, 4); + push(4, 5); + push(5, 6); + EXPECT_EQ(6, pop(5).value()); + EXPECT_EQ(5, pop(4).value()); + EXPECT_EQ(2, pop().value()); + EXPECT_EQ(3, pop().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +class QueueMapPeekTest: public QueueMapTest {}; + +TEST_F(QueueMapPeekTest, PoppingFromEmpty) { + EXPECT_EQ(boost::none, peek()); +} + +TEST_F(QueueMapPeekTest, PushingOne) { + push(3, 2); + EXPECT_EQ(2, peek().value()); +} + +TEST_F(QueueMapPeekTest, PushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_EQ(3, peek().value()); + EXPECT_EQ(3, peek().value()); + EXPECT_EQ(3, pop().value()); + EXPECT_EQ(4, peek().value()); + EXPECT_EQ(4, peek().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(boost::none, peek()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapPeekTest, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, peek().value()); +} + +class CopyableValueType { +public: + static int numCopyConstructorCalled; + CopyableValueType(int value): _value(value) {} + CopyableValueType(const CopyableValueType &rhs): CopyableValueType(rhs._value) { + ++numCopyConstructorCalled; + } + CopyableValueType(CopyableValueType &&rhs): CopyableValueType(rhs._value) { + //Don't increase numCopyConstructorCalled + } + int value() const { + return _value; + } +private: + int _value; +}; +int CopyableValueType::numCopyConstructorCalled = 0; + +//Test that QueueMap uses a move constructor for Value if possible +class QueueMapMoveConstructorTest: public Test { +public: + QueueMapMoveConstructorTest() { + CopyableValueType::numCopyConstructorCalled = 0; + map = make_unique>(); + } + unique_ptr> map; +}; + +TEST_F(QueueMapMoveConstructorTest, PushingAndPopping_MoveIntoMap) { + map->push(MinimalKeyType::create(0), CopyableValueType(2)); + CopyableValueType val = map->pop().value(); + EXPECT_EQ(0, CopyableValueType::numCopyConstructorCalled); +} + +TEST_F(QueueMapMoveConstructorTest, PushingAndPoppingPerKey_MoveIntoMap) { + map->push(MinimalKeyType::create(0), CopyableValueType(2)); + CopyableValueType val = map->pop(MinimalKeyType::create(0)).value(); + EXPECT_EQ(0, CopyableValueType::numCopyConstructorCalled); +} + +TEST_F(QueueMapMoveConstructorTest, PushingAndPopping_CopyIntoMap) { + CopyableValueType value(2); + map->push(MinimalKeyType::create(0), value); + CopyableValueType val = map->pop().value(); + EXPECT_EQ(1, CopyableValueType::numCopyConstructorCalled); +} + +TEST_F(QueueMapMoveConstructorTest, PushingAndPoppingPerKey_CopyIntoMap) { + CopyableValueType value(2); + map->push(MinimalKeyType::create(0), value); + CopyableValueType val = map->pop(MinimalKeyType::create(0)).value(); + EXPECT_EQ(1, CopyableValueType::numCopyConstructorCalled); } //TODO Pushing the same key twice -//TODO Popping from empty -//TODO Popping invalid key -//TODO Test that in all cases, destructors of Value are called correctly in QueueMap when [a] pop() [b] pop(key) [c] ~QueueMap() -//TODO Test that pushing and popping a copy-and-move-constructible object only called 1 copy constructor -//TODO Test that pushing and popping doesn't invalidate objects (e.g, calls too many destructors) From bc33f6987732e293b4a03c104b38d49fabd61ea5 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 27 Apr 2015 10:47:33 +0200 Subject: [PATCH 23/41] TODOs --- implementations/caching/Cache.cpp | 2 +- implementations/caching/QueueMap.h | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/implementations/caching/Cache.cpp b/implementations/caching/Cache.cpp index e4cbac53..45b5b281 100644 --- a/implementations/caching/Cache.cpp +++ b/implementations/caching/Cache.cpp @@ -48,7 +48,7 @@ void Cache::push(unique_ptr block) { void Cache::_popOldEntries() { lock_guard lock(_mutex); while(_cachedBlocks.size() > 0 && _cachedBlocks.peek()->ageSeconds() > PURGE_LIFETIME_SEC) { - _cachedBlocks.pop(); + _cachedBlocks.pop(); } } diff --git a/implementations/caching/QueueMap.h b/implementations/caching/QueueMap.h index c8814691..e3cbb0ec 100644 --- a/implementations/caching/QueueMap.h +++ b/implementations/caching/QueueMap.h @@ -11,8 +11,7 @@ namespace blockstore { namespace caching { -//TODO Test -//TODO Move to utils +// A class that is a queue and a map at the same time. We could also see it as an addressable queue. template class QueueMap { public: From bf77e0ffa7d3d72f82cab7a85668e4a783455a43 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 27 Apr 2015 17:58:34 +0200 Subject: [PATCH 24/41] Small fix in AES256_GCM --- implementations/encrypted/EncryptedBlockStore.h | 7 +------ implementations/encrypted/ciphers/AES256_GCM.cpp | 3 ++- test/implementations/encrypted/EncryptedBlockStoreTest.cpp | 6 +++--- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/implementations/encrypted/EncryptedBlockStore.h b/implementations/encrypted/EncryptedBlockStore.h index 9fb01c84..09c35723 100644 --- a/implementations/encrypted/EncryptedBlockStore.h +++ b/implementations/encrypted/EncryptedBlockStore.h @@ -52,12 +52,7 @@ std::unique_ptr EncryptedBlockStore::load(const Key &key) { if (block.get() == nullptr) { return nullptr; } - auto encBlock = EncryptedBlock::TryDecrypt(std::move(block), _encKey); - if (encBlock.get() == nullptr) { - //TODO Think about how to do logging - - } - return std::move(encBlock); + return EncryptedBlock::TryDecrypt(std::move(block), _encKey); } template diff --git a/implementations/encrypted/ciphers/AES256_GCM.cpp b/implementations/encrypted/ciphers/AES256_GCM.cpp index d985c786..4ccf400a 100644 --- a/implementations/encrypted/ciphers/AES256_GCM.cpp +++ b/implementations/encrypted/ciphers/AES256_GCM.cpp @@ -47,7 +47,8 @@ boost::optional AES256_GCM::decrypt(const byte *ciphertext, unsigned int c try { ArraySource((byte*)ciphertextData, ciphertextSize - IV_SIZE, true, new AuthenticatedDecryptionFilter(decryption, - new ArraySink((byte*)plaintext.data(), plaintext.size()) + new ArraySink((byte*)plaintext.data(), plaintext.size()), + AuthenticatedDecryptionFilter::DEFAULT_FLAGS, TAG_SIZE ) ); return std::move(plaintext); diff --git a/test/implementations/encrypted/EncryptedBlockStoreTest.cpp b/test/implementations/encrypted/EncryptedBlockStoreTest.cpp index dd5479ac..13c151a1 100644 --- a/test/implementations/encrypted/EncryptedBlockStoreTest.cpp +++ b/test/implementations/encrypted/EncryptedBlockStoreTest.cpp @@ -1,4 +1,4 @@ -#include "../../../implementations/encrypted/ciphers/AES256_CFB.h" +#include "../../../implementations/encrypted/ciphers/AES256_GCM.h" #include "../../../implementations/encrypted/EncryptedBlockStore.h" #include "../../../implementations/testfake/FakeBlockStore.h" #include "../../testutils/BlockStoreTest.h" @@ -8,7 +8,7 @@ using blockstore::BlockStore; using blockstore::encrypted::EncryptedBlockStore; using blockstore::testfake::FakeBlockStore; -using blockstore::encrypted::AES256_CFB; +using blockstore::encrypted::AES256_GCM; using std::unique_ptr; using std::make_unique; @@ -16,7 +16,7 @@ using std::make_unique; class EncryptedBlockStoreTestFixture: public BlockStoreTestFixture { public: unique_ptr createBlockStore() override { - return make_unique>(make_unique(), AES256_CFB::EncryptionKey::FromString("1491BB4932A389EE14BC7090A272EE5517627CFA147A971A8E6E747E0C772972")); + return make_unique>(make_unique(), AES256_GCM::EncryptionKey::FromString("1491BB4932A389EE14BC7090A272EE5517627CFA147A971A8E6E747E0C772972")); } }; From 58f43f69293a16ceaac51adea42b27e37854b8bb Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 27 Apr 2015 18:20:44 +0200 Subject: [PATCH 25/41] Remove unneeded field in cache entry --- implementations/caching/CacheEntry.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/implementations/caching/CacheEntry.h b/implementations/caching/CacheEntry.h index 7c11f0c0..cbe03af8 100644 --- a/implementations/caching/CacheEntry.h +++ b/implementations/caching/CacheEntry.h @@ -26,18 +26,9 @@ public: return std::move(_block); } - void setNextEntry(const CacheEntry *entry) { - _nextEntry = entry; - } - - const CacheEntry *nextEntry() { - return _nextEntry; - } - private: boost::posix_time::ptime _lastAccess; std::unique_ptr _block; - const CacheEntry *_nextEntry; static boost::posix_time::ptime currentTime() { return boost::posix_time::microsec_clock::local_time(); From 211083137425e9016b0bbc125d94f92d69990c4b Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 27 Apr 2015 18:20:51 +0200 Subject: [PATCH 26/41] Make constructors explicit where adequate --- implementations/caching/Cache.cpp | 2 +- implementations/caching/CacheEntry.h | 2 +- implementations/caching/CachingBlockStore.h | 2 +- implementations/ondisk/FileAlreadyExistsException.h | 3 ++- implementations/ondisk/OnDiskBlockStore.h | 2 +- implementations/parallelaccess/BlockRef.h | 2 +- implementations/parallelaccess/ParallelAccessBlockStore.h | 2 +- .../parallelaccess/ParallelAccessBlockStoreAdapter.h | 2 +- utils/FileDoesntExistException.h | 2 +- 9 files changed, 10 insertions(+), 9 deletions(-) diff --git a/implementations/caching/Cache.cpp b/implementations/caching/Cache.cpp index 45b5b281..0a5943a4 100644 --- a/implementations/caching/Cache.cpp +++ b/implementations/caching/Cache.cpp @@ -42,7 +42,7 @@ void Cache::push(unique_ptr block) { assert(_cachedBlocks.size() == MAX_ENTRIES-1); } Key key = block->key(); - _cachedBlocks.push(key, std::move(block)); + _cachedBlocks.push(key, CacheEntry(std::move(block))); } void Cache::_popOldEntries() { diff --git a/implementations/caching/CacheEntry.h b/implementations/caching/CacheEntry.h index cbe03af8..5ca2c6bc 100644 --- a/implementations/caching/CacheEntry.h +++ b/implementations/caching/CacheEntry.h @@ -13,7 +13,7 @@ namespace caching { class CacheEntry { public: - CacheEntry(std::unique_ptr block): _lastAccess(currentTime()), _block(std::move(block)) { + explicit CacheEntry(std::unique_ptr block): _lastAccess(currentTime()), _block(std::move(block)) { } CacheEntry(CacheEntry &&) = default; diff --git a/implementations/caching/CachingBlockStore.h b/implementations/caching/CachingBlockStore.h index ae298f24..a30eb4c7 100644 --- a/implementations/caching/CachingBlockStore.h +++ b/implementations/caching/CachingBlockStore.h @@ -11,7 +11,7 @@ namespace caching { //TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel) class CachingBlockStore: public BlockStore { public: - CachingBlockStore(std::unique_ptr baseBlockStore); + explicit CachingBlockStore(std::unique_ptr baseBlockStore); Key createKey() override; std::unique_ptr tryCreate(const Key &key, cpputils::Data data) override; diff --git a/implementations/ondisk/FileAlreadyExistsException.h b/implementations/ondisk/FileAlreadyExistsException.h index 2a23fdd1..85cf18e5 100644 --- a/implementations/ondisk/FileAlreadyExistsException.h +++ b/implementations/ondisk/FileAlreadyExistsException.h @@ -9,9 +9,10 @@ namespace blockstore { namespace ondisk { +//TODO We probably don't want an exception for that class FileAlreadyExistsException: public std::runtime_error { public: - FileAlreadyExistsException(const boost::filesystem::path &filepath); + explicit FileAlreadyExistsException(const boost::filesystem::path &filepath); virtual ~FileAlreadyExistsException(); }; diff --git a/implementations/ondisk/OnDiskBlockStore.h b/implementations/ondisk/OnDiskBlockStore.h index 98ba7163..af7f72fc 100644 --- a/implementations/ondisk/OnDiskBlockStore.h +++ b/implementations/ondisk/OnDiskBlockStore.h @@ -12,7 +12,7 @@ namespace ondisk { class OnDiskBlockStore: public BlockStoreWithRandomKeys { public: - OnDiskBlockStore(const boost::filesystem::path &rootdir); + explicit OnDiskBlockStore(const boost::filesystem::path &rootdir); std::unique_ptr tryCreate(const Key &key, cpputils::Data data) override; std::unique_ptr load(const Key &key) override; diff --git a/implementations/parallelaccess/BlockRef.h b/implementations/parallelaccess/BlockRef.h index a379ef7f..588b3c6e 100644 --- a/implementations/parallelaccess/BlockRef.h +++ b/implementations/parallelaccess/BlockRef.h @@ -14,7 +14,7 @@ class ParallelAccessBlockStore; class BlockRef: public Block, public parallelaccessstore::ParallelAccessStore::ResourceRefBase { public: //TODO Unneccessarily storing Key twice here (in parent class and in _baseBlock). - BlockRef(Block *baseBlock): Block(baseBlock->key()), _baseBlock(baseBlock) {} + explicit BlockRef(Block *baseBlock): Block(baseBlock->key()), _baseBlock(baseBlock) {} const void *data() const override { return _baseBlock->data(); diff --git a/implementations/parallelaccess/ParallelAccessBlockStore.h b/implementations/parallelaccess/ParallelAccessBlockStore.h index 8ba08ba6..2f896b09 100644 --- a/implementations/parallelaccess/ParallelAccessBlockStore.h +++ b/implementations/parallelaccess/ParallelAccessBlockStore.h @@ -12,7 +12,7 @@ namespace parallelaccess { //TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel) class ParallelAccessBlockStore: public BlockStore { public: - ParallelAccessBlockStore(std::unique_ptr baseBlockStore); + explicit ParallelAccessBlockStore(std::unique_ptr baseBlockStore); Key createKey() override; std::unique_ptr tryCreate(const Key &key, cpputils::Data data) override; diff --git a/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.h b/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.h index 56c9d7de..d4784f16 100644 --- a/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.h +++ b/implementations/parallelaccess/ParallelAccessBlockStoreAdapter.h @@ -10,7 +10,7 @@ namespace parallelaccess { class ParallelAccessBlockStoreAdapter: public parallelaccessstore::ParallelAccessBaseStore { public: - ParallelAccessBlockStoreAdapter(BlockStore *baseBlockStore) + explicit ParallelAccessBlockStoreAdapter(BlockStore *baseBlockStore) :_baseBlockStore(std::move(baseBlockStore)) { } diff --git a/utils/FileDoesntExistException.h b/utils/FileDoesntExistException.h index d4c0a521..9ae623db 100644 --- a/utils/FileDoesntExistException.h +++ b/utils/FileDoesntExistException.h @@ -10,7 +10,7 @@ namespace blockstore { class FileDoesntExistException: public std::runtime_error { public: - FileDoesntExistException(const boost::filesystem::path &filepath); + explicit FileDoesntExistException(const boost::filesystem::path &filepath); virtual ~FileDoesntExistException(); }; From f286886b4903df1a3d7c7923fc7395e9c22a9c8f Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 27 Apr 2015 22:11:15 +0200 Subject: [PATCH 27/41] Handle QueueMap::push(existing_key) better and write more test cases for QueueMap --- implementations/caching/QueueMap.h | 4 ++- test/implementations/caching/QueueMapTest.cpp | 30 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/implementations/caching/QueueMap.h b/implementations/caching/QueueMap.h index e3cbb0ec..f412432b 100644 --- a/implementations/caching/QueueMap.h +++ b/implementations/caching/QueueMap.h @@ -25,7 +25,9 @@ public: void push(const Key &key, Value value) { auto newEntry = _entries.emplace(std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple(_sentinel.prev, &_sentinel)); - assert(newEntry.second == true); + if(newEntry.second != true) { + throw std::logic_error("There is already an element with this key"); + } newEntry.first->second.init(&newEntry.first->first, std::move(value)); //The following is ok, because std::unordered_map never invalidates pointers to its entries _sentinel.prev->next = &newEntry.first->second; diff --git a/test/implementations/caching/QueueMapTest.cpp b/test/implementations/caching/QueueMapTest.cpp index 851180e9..c49e6889 100644 --- a/test/implementations/caching/QueueMapTest.cpp +++ b/test/implementations/caching/QueueMapTest.cpp @@ -398,6 +398,34 @@ TEST_F(QueueMapValueTest, PushingSomePoppingLastPerKey) { EXPECT_EQ(boost::none, pop()); } +//This test forces the underlying datastructure (std::map or std::unordered_map) to grow and reallocate memory. +//So it tests, that QueueMap still works after reallocating memory. +TEST_F(QueueMapValueTest, ManyValues) { + //Push 1 million entries + for (int i = 0; i < 1000000; ++i) { + push(i, 2*i); + } + //pop every other one by key + for (int i = 0; i < 1000000; i += 2) { + EXPECT_EQ(2*i, pop(i).value()); + } + //pop the rest in queue order + for (int i = 1; i < 1000000; i += 2) { + EXPECT_EQ(2*i, peek().value()); + EXPECT_EQ(2*i, pop().value()); + } + EXPECT_EQ(0, size()); + EXPECT_EQ(boost::none, pop()); + EXPECT_EQ(boost::none, peek()); +} + +TEST_F(QueueMapValueTest, PushAlreadyExistingValue) { + push(2, 3); + EXPECT_ANY_THROW( + push(2, 4); + ); +} + class QueueMapPeekTest: public QueueMapTest {}; TEST_F(QueueMapPeekTest, PoppingFromEmpty) { @@ -483,5 +511,3 @@ TEST_F(QueueMapMoveConstructorTest, PushingAndPoppingPerKey_CopyIntoMap) { CopyableValueType val = map->pop(MinimalKeyType::create(0)).value(); EXPECT_EQ(1, CopyableValueType::numCopyConstructorCalled); } - -//TODO Pushing the same key twice From 1ff807e09c181707c6251169444a06f33f7b256f Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 27 Apr 2015 22:46:57 +0200 Subject: [PATCH 28/41] Make Cache a generic over Key/Value. Furthermore, add test skeleton for Cache --- implementations/caching/Cache.cpp | 56 ------------ implementations/caching/Cache.h | 42 --------- implementations/caching/CacheEntry.cpp | 2 - implementations/caching/CachingBlockStore.cpp | 20 +++-- implementations/caching/CachingBlockStore.h | 4 +- implementations/caching/cache/Cache.cpp | 1 + implementations/caching/cache/Cache.h | 88 +++++++++++++++++++ implementations/caching/cache/CacheEntry.cpp | 1 + .../caching/{ => cache}/CacheEntry.h | 10 +-- .../caching/{ => cache}/PeriodicTask.cpp | 0 .../caching/{ => cache}/PeriodicTask.h | 0 .../caching/{ => cache}/QueueMap.cpp | 0 .../caching/{ => cache}/QueueMap.h | 0 .../caching/cache/CacheTest.cpp | 12 +++ .../caching/{ => cache}/PeriodicTaskTest.cpp | 2 +- .../caching/{ => cache}/QueueMapTest.cpp | 2 +- 16 files changed, 123 insertions(+), 117 deletions(-) delete mode 100644 implementations/caching/Cache.cpp delete mode 100644 implementations/caching/Cache.h delete mode 100644 implementations/caching/CacheEntry.cpp create mode 100644 implementations/caching/cache/Cache.cpp create mode 100644 implementations/caching/cache/Cache.h create mode 100644 implementations/caching/cache/CacheEntry.cpp rename implementations/caching/{ => cache}/CacheEntry.h (76%) rename implementations/caching/{ => cache}/PeriodicTask.cpp (100%) rename implementations/caching/{ => cache}/PeriodicTask.h (100%) rename implementations/caching/{ => cache}/QueueMap.cpp (100%) rename implementations/caching/{ => cache}/QueueMap.h (100%) create mode 100644 test/implementations/caching/cache/CacheTest.cpp rename test/implementations/caching/{ => cache}/PeriodicTaskTest.cpp (95%) rename test/implementations/caching/{ => cache}/QueueMapTest.cpp (99%) diff --git a/implementations/caching/Cache.cpp b/implementations/caching/Cache.cpp deleted file mode 100644 index 0a5943a4..00000000 --- a/implementations/caching/Cache.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "Cache.h" -#include "PeriodicTask.h" - -using std::unique_ptr; -using std::make_unique; -using std::mutex; -using std::lock_guard; -using std::pair; - -namespace blockstore { -namespace caching { - -constexpr uint32_t Cache::MAX_ENTRIES; -constexpr double Cache::PURGE_LIFETIME_SEC; -constexpr double Cache::PURGE_INTERVAL; -constexpr double Cache::MAX_LIFETIME_SEC; - -Cache::Cache(): _cachedBlocks(), _timeoutFlusher(nullptr) { - //Don't initialize timeoutFlusher in the initializer list, - //because it then might already call Cache::popOldEntries() before Cache is done constructing - _timeoutFlusher = make_unique(std::bind(&Cache::_popOldEntries, this), PURGE_INTERVAL); -} - -Cache::~Cache() { -} - -unique_ptr Cache::pop(const Key &key) { - lock_guard lock(_mutex); - auto found = _cachedBlocks.pop(key); - if (!found) { - return nullptr; - } - auto block = found->releaseBlock(); - return block; -} - -void Cache::push(unique_ptr block) { - lock_guard lock(_mutex); - assert(_cachedBlocks.size() <= MAX_ENTRIES); - if (_cachedBlocks.size() == MAX_ENTRIES) { - _cachedBlocks.pop(); - assert(_cachedBlocks.size() == MAX_ENTRIES-1); - } - Key key = block->key(); - _cachedBlocks.push(key, CacheEntry(std::move(block))); -} - -void Cache::_popOldEntries() { - lock_guard lock(_mutex); - while(_cachedBlocks.size() > 0 && _cachedBlocks.peek()->ageSeconds() > PURGE_LIFETIME_SEC) { - _cachedBlocks.pop(); - } -} - -} -} diff --git a/implementations/caching/Cache.h b/implementations/caching/Cache.h deleted file mode 100644 index 8ff93ae6..00000000 --- a/implementations/caching/Cache.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once -#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_H_ -#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_H_ - -#include "CacheEntry.h" -#include "QueueMap.h" -#include "../../interface/Block.h" -#include -#include - -namespace blockstore { -namespace caching { -class PeriodicTask; - -//TODO Test - -class Cache { -public: - static constexpr uint32_t MAX_ENTRIES = 1000; - //TODO Experiment with good values - static constexpr double PURGE_LIFETIME_SEC = 0.5; //When an entry has this age, it will be purged from the cache - static constexpr double PURGE_INTERVAL = 0.5; // With this interval, we check for entries to purge - static constexpr double MAX_LIFETIME_SEC = PURGE_LIFETIME_SEC + PURGE_INTERVAL; // This is the oldest age an entry can reach (given purging works in an ideal world, i.e. with the ideal interval and in zero time) - - Cache(); - virtual ~Cache(); - - void push(std::unique_ptr block); - std::unique_ptr pop(const Key &key); - -private: - void _popOldEntries(); - - mutable std::mutex _mutex; - QueueMap _cachedBlocks; - std::unique_ptr _timeoutFlusher; -}; - -} -} - -#endif diff --git a/implementations/caching/CacheEntry.cpp b/implementations/caching/CacheEntry.cpp deleted file mode 100644 index 0ca41a63..00000000 --- a/implementations/caching/CacheEntry.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#include "CacheEntry.h" -#include "../../interface/Block.h" diff --git a/implementations/caching/CachingBlockStore.cpp b/implementations/caching/CachingBlockStore.cpp index 6518d80c..67d841ea 100644 --- a/implementations/caching/CachingBlockStore.cpp +++ b/implementations/caching/CachingBlockStore.cpp @@ -28,13 +28,16 @@ unique_ptr CachingBlockStore::tryCreate(const Key &key, Data data) { } unique_ptr CachingBlockStore::load(const Key &key) { - auto block = _cache.pop(key); - if (block.get() != nullptr) { - return make_unique(std::move(block), this); - } - block = _baseBlockStore->load(key); - if (block.get() == nullptr) { - return nullptr; + boost::optional> optBlock = _cache.pop(key); + unique_ptr block; + //TODO an optional<> class with .getOrElse() would make this code simpler. boost::optional<>::value_or_eval didn't seem to work with unique_ptr members. + if (optBlock) { + block = std::move(*optBlock); + } else { + block = _baseBlockStore->load(key); + if (block.get() == nullptr) { + return nullptr; + } } return make_unique(std::move(block), this); } @@ -57,7 +60,8 @@ uint64_t CachingBlockStore::numBlocks() const { } void CachingBlockStore::release(unique_ptr block) { - _cache.push(std::move(block)); + Key key = block->key(); + _cache.push(key, std::move(block)); } std::unique_ptr CachingBlockStore::tryCreateInBaseStore(const Key &key, Data data) { diff --git a/implementations/caching/CachingBlockStore.h b/implementations/caching/CachingBlockStore.h index a30eb4c7..073ecd97 100644 --- a/implementations/caching/CachingBlockStore.h +++ b/implementations/caching/CachingBlockStore.h @@ -2,7 +2,7 @@ #ifndef BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_ #define BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_ -#include "Cache.h" +#include "cache/Cache.h" #include "../../interface/BlockStore.h" namespace blockstore { @@ -26,7 +26,7 @@ public: private: std::unique_ptr _baseBlockStore; - Cache _cache; + Cache> _cache; uint32_t _numNewBlocks; DISALLOW_COPY_AND_ASSIGN(CachingBlockStore); diff --git a/implementations/caching/cache/Cache.cpp b/implementations/caching/cache/Cache.cpp new file mode 100644 index 00000000..2d464949 --- /dev/null +++ b/implementations/caching/cache/Cache.cpp @@ -0,0 +1 @@ +#include "Cache.h" diff --git a/implementations/caching/cache/Cache.h b/implementations/caching/cache/Cache.h new file mode 100644 index 00000000..f47e6c95 --- /dev/null +++ b/implementations/caching/cache/Cache.h @@ -0,0 +1,88 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_H_ +#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHE_H_ + +#include "CacheEntry.h" +#include "QueueMap.h" +#include "PeriodicTask.h" +#include +#include +#include + +namespace blockstore { +namespace caching { + +//TODO Test + +template +class Cache { +public: + static constexpr uint32_t MAX_ENTRIES = 1000; + //TODO Experiment with good values + static constexpr double PURGE_LIFETIME_SEC = 0.5; //When an entry has this age, it will be purged from the cache + static constexpr double PURGE_INTERVAL = 0.5; // With this interval, we check for entries to purge + static constexpr double MAX_LIFETIME_SEC = PURGE_LIFETIME_SEC + PURGE_INTERVAL; // This is the oldest age an entry can reach (given purging works in an ideal world, i.e. with the ideal interval and in zero time) + + Cache(); + virtual ~Cache(); + + void push(const Key &key, Value value); + boost::optional pop(const Key &key); + +private: + void _popOldEntries(); + + mutable std::mutex _mutex; + QueueMap> _cachedBlocks; + std::unique_ptr _timeoutFlusher; +}; + +template constexpr uint32_t Cache::MAX_ENTRIES; +template constexpr double Cache::PURGE_LIFETIME_SEC; +template constexpr double Cache::PURGE_INTERVAL; +template constexpr double Cache::MAX_LIFETIME_SEC; + +template +Cache::Cache(): _cachedBlocks(), _timeoutFlusher(nullptr) { + //Don't initialize timeoutFlusher in the initializer list, + //because it then might already call Cache::popOldEntries() before Cache is done constructing + _timeoutFlusher = std::make_unique(std::bind(&Cache::_popOldEntries, this), PURGE_INTERVAL); +} + +template +Cache::~Cache() { +} + +template +boost::optional Cache::pop(const Key &key) { + std::lock_guard lock(_mutex); + auto found = _cachedBlocks.pop(key); + if (!found) { + return boost::none; + } + return found->releaseValue(); +} + +template +void Cache::push(const Key &key, Value value) { + std::lock_guard lock(_mutex); + assert(_cachedBlocks.size() <= MAX_ENTRIES); + if (_cachedBlocks.size() == MAX_ENTRIES) { + _cachedBlocks.pop(); + assert(_cachedBlocks.size() == MAX_ENTRIES-1); + } + _cachedBlocks.push(key, CacheEntry(std::move(value))); +} + +template +void Cache::_popOldEntries() { + std::lock_guard lock(_mutex); + while(_cachedBlocks.size() > 0 && _cachedBlocks.peek()->ageSeconds() > PURGE_LIFETIME_SEC) { + _cachedBlocks.pop(); + } +} + +} +} + +#endif diff --git a/implementations/caching/cache/CacheEntry.cpp b/implementations/caching/cache/CacheEntry.cpp new file mode 100644 index 00000000..a3420ac8 --- /dev/null +++ b/implementations/caching/cache/CacheEntry.cpp @@ -0,0 +1 @@ +#include "CacheEntry.h" diff --git a/implementations/caching/CacheEntry.h b/implementations/caching/cache/CacheEntry.h similarity index 76% rename from implementations/caching/CacheEntry.h rename to implementations/caching/cache/CacheEntry.h index 5ca2c6bc..b49ba970 100644 --- a/implementations/caching/CacheEntry.h +++ b/implementations/caching/cache/CacheEntry.h @@ -8,12 +8,12 @@ #include namespace blockstore { -class Block; namespace caching { +template class CacheEntry { public: - explicit CacheEntry(std::unique_ptr block): _lastAccess(currentTime()), _block(std::move(block)) { + explicit CacheEntry(Value value): _lastAccess(currentTime()), _value(std::move(value)) { } CacheEntry(CacheEntry &&) = default; @@ -22,13 +22,13 @@ public: return ((double)(currentTime() - _lastAccess).total_nanoseconds()) / ((double)1000000000); } - std::unique_ptr releaseBlock() { - return std::move(_block); + Value releaseValue() { + return std::move(_value); } private: boost::posix_time::ptime _lastAccess; - std::unique_ptr _block; + Value _value; static boost::posix_time::ptime currentTime() { return boost::posix_time::microsec_clock::local_time(); diff --git a/implementations/caching/PeriodicTask.cpp b/implementations/caching/cache/PeriodicTask.cpp similarity index 100% rename from implementations/caching/PeriodicTask.cpp rename to implementations/caching/cache/PeriodicTask.cpp diff --git a/implementations/caching/PeriodicTask.h b/implementations/caching/cache/PeriodicTask.h similarity index 100% rename from implementations/caching/PeriodicTask.h rename to implementations/caching/cache/PeriodicTask.h diff --git a/implementations/caching/QueueMap.cpp b/implementations/caching/cache/QueueMap.cpp similarity index 100% rename from implementations/caching/QueueMap.cpp rename to implementations/caching/cache/QueueMap.cpp diff --git a/implementations/caching/QueueMap.h b/implementations/caching/cache/QueueMap.h similarity index 100% rename from implementations/caching/QueueMap.h rename to implementations/caching/cache/QueueMap.h diff --git a/test/implementations/caching/cache/CacheTest.cpp b/test/implementations/caching/cache/CacheTest.cpp new file mode 100644 index 00000000..e746baaa --- /dev/null +++ b/test/implementations/caching/cache/CacheTest.cpp @@ -0,0 +1,12 @@ +#include + +#include "../../../../implementations/caching/cache/Cache.h" + +using ::testing::Test; + +using namespace blockstore::caching; + +class CacheTest: public Test { +public: + Cache cache; +}; diff --git a/test/implementations/caching/PeriodicTaskTest.cpp b/test/implementations/caching/cache/PeriodicTaskTest.cpp similarity index 95% rename from test/implementations/caching/PeriodicTaskTest.cpp rename to test/implementations/caching/cache/PeriodicTaskTest.cpp index 4e945961..d49492ad 100644 --- a/test/implementations/caching/PeriodicTaskTest.cpp +++ b/test/implementations/caching/cache/PeriodicTaskTest.cpp @@ -1,6 +1,6 @@ #include -#include "../../../implementations/caching/PeriodicTask.h" +#include "../../../../implementations/caching/cache/PeriodicTask.h" #include #include diff --git a/test/implementations/caching/QueueMapTest.cpp b/test/implementations/caching/cache/QueueMapTest.cpp similarity index 99% rename from test/implementations/caching/QueueMapTest.cpp rename to test/implementations/caching/cache/QueueMapTest.cpp index c49e6889..1ba2c368 100644 --- a/test/implementations/caching/QueueMapTest.cpp +++ b/test/implementations/caching/cache/QueueMapTest.cpp @@ -1,6 +1,6 @@ #include #include -#include "../../../implementations/caching/QueueMap.h" +#include "../../../../implementations/caching/cache/QueueMap.h" #include #include From c47434d3b30acb171bd6e2562c7a4542a8722da8 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 27 Apr 2015 23:09:29 +0200 Subject: [PATCH 29/41] Separated QueueMapTest.cpp into test groups with their own files each --- .../caching/cache/QueueMapTest.cpp | 513 ------------------ .../caching/cache/QueueMapTest_MemoryLeak.cpp | 87 +++ .../cache/QueueMapTest_MoveConstructor.cpp | 64 +++ .../caching/cache/QueueMapTest_Peek.cpp | 34 ++ .../caching/cache/QueueMapTest_Size.cpp | 79 +++ .../caching/cache/QueueMapTest_Values.cpp | 150 +++++ .../cache/testutils/MinimalKeyType.cpp | 3 + .../caching/cache/testutils/MinimalKeyType.h | 47 ++ .../cache/testutils/MinimalValueType.cpp | 3 + .../cache/testutils/MinimalValueType.h | 44 ++ .../caching/cache/testutils/QueueMapTest.cpp | 45 ++ .../caching/cache/testutils/QueueMapTest.h | 32 ++ 12 files changed, 588 insertions(+), 513 deletions(-) delete mode 100644 test/implementations/caching/cache/QueueMapTest.cpp create mode 100644 test/implementations/caching/cache/QueueMapTest_MemoryLeak.cpp create mode 100644 test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp create mode 100644 test/implementations/caching/cache/QueueMapTest_Peek.cpp create mode 100644 test/implementations/caching/cache/QueueMapTest_Size.cpp create mode 100644 test/implementations/caching/cache/QueueMapTest_Values.cpp create mode 100644 test/implementations/caching/cache/testutils/MinimalKeyType.cpp create mode 100644 test/implementations/caching/cache/testutils/MinimalKeyType.h create mode 100644 test/implementations/caching/cache/testutils/MinimalValueType.cpp create mode 100644 test/implementations/caching/cache/testutils/MinimalValueType.h create mode 100644 test/implementations/caching/cache/testutils/QueueMapTest.cpp create mode 100644 test/implementations/caching/cache/testutils/QueueMapTest.h diff --git a/test/implementations/caching/cache/QueueMapTest.cpp b/test/implementations/caching/cache/QueueMapTest.cpp deleted file mode 100644 index 1ba2c368..00000000 --- a/test/implementations/caching/cache/QueueMapTest.cpp +++ /dev/null @@ -1,513 +0,0 @@ -#include -#include -#include "../../../../implementations/caching/cache/QueueMap.h" -#include -#include - -using ::testing::Test; -using std::unique_ptr; -using std::make_unique; - -using namespace blockstore::caching; - -// This is a not-default-constructible Key type -class MinimalKeyType { -public: - static int instances; - static MinimalKeyType create(int value) { - return MinimalKeyType(value); - } - bool operator==(const MinimalKeyType &rhs) const { - return _value == rhs._value; - } - int value() const { - return _value; - } - MinimalKeyType(const MinimalKeyType &rhs): MinimalKeyType(rhs.value()) { - } - ~MinimalKeyType() { - --instances; - } -private: - MinimalKeyType(int value): _value(value) { - ++instances; - } - int _value; -}; -int MinimalKeyType::instances = 0; -namespace std { -template <> struct hash { - size_t operator()(const MinimalKeyType &obj) const { - return obj.value(); - } -}; -} -// This is a not-default-constructible non-copyable but moveable Value type -class MinimalValueType { -public: - static int instances; - static MinimalValueType create(int value) { - return MinimalValueType(value); - } - MinimalValueType(MinimalValueType &&rhs): MinimalValueType(rhs.value()) { - rhs._isMoved = true; - } - ~MinimalValueType() { - assert(!_isDestructed); - --instances; - _isDestructed = true; - } - int value() const { - assert(!_isMoved && !_isDestructed); - return _value; - } -private: - MinimalValueType(int value): _value(value), _isMoved(false), _isDestructed(false) { - ++instances; - } - int _value; - bool _isMoved; - bool _isDestructed; - DISALLOW_COPY_AND_ASSIGN(MinimalValueType); -}; -int MinimalValueType::instances = 0; - -class QueueMapTest: public Test { -public: - QueueMapTest() { - MinimalKeyType::instances = 0; - MinimalValueType::instances = 0; - _map = make_unique>(); - } - ~QueueMapTest() { - _map.reset(); - EXPECT_EQ(0, MinimalKeyType::instances); - EXPECT_EQ(0, MinimalValueType::instances); - } - void push(int key, int value) { - _map->push(MinimalKeyType::create(key), MinimalValueType::create(value)); - } - boost::optional pop() { - auto elem = _map->pop(); - if (!elem) { - return boost::none; - } - return elem.value().value(); - } - boost::optional pop(int key) { - auto elem = _map->pop(MinimalKeyType::create(key)); - if (!elem) { - return boost::none; - } - return elem.value().value(); - } - boost::optional peek() { - auto elem = _map->peek(); - if (!elem) { - return boost::none; - } - return elem.value().value(); - } - int size() { - return _map->size(); - } -private: - unique_ptr> _map; -}; - -class QueueMapSizeTest: public QueueMapTest {}; - -TEST_F(QueueMapSizeTest, Empty) { - EXPECT_EQ(0, size()); -} - -TEST_F(QueueMapSizeTest, AfterPushingOne) { - push(2, 3); - EXPECT_EQ(1, size()); -} - -TEST_F(QueueMapSizeTest, AfterPushingTwo) { - push(2, 3); - push(3, 4); - EXPECT_EQ(2, size()); -} - -TEST_F(QueueMapSizeTest, AfterPushingTwoAndPoppingOldest) { - push(2, 3); - push(3, 4); - pop(); - EXPECT_EQ(1, size()); -} - -TEST_F(QueueMapSizeTest, AfterPushingTwoAndPoppingFirst) { - push(2, 3); - push(3, 4); - pop(2); - EXPECT_EQ(1, size()); -} - -TEST_F(QueueMapSizeTest, AfterPushingTwoAndPoppingLast) { - push(2, 3); - push(3, 4); - pop(3); - EXPECT_EQ(1, size()); -} - -TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOne) { - push(2, 3); - pop(); - EXPECT_EQ(0, size()); -} - -TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOnePerKey) { - push(2, 3); - pop(2); - EXPECT_EQ(0, size()); -} - -TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOnePushingOne) { - push(2, 3); - pop(); - push(3, 4); - EXPECT_EQ(1, size()); -} - -TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOnePerKeyPushingOne) { - push(2, 3); - pop(2); - push(3, 4); - EXPECT_EQ(1, size()); -} - -TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOnePushingSame) { - push(2, 3); - pop(); - push(2, 3); - EXPECT_EQ(1, size()); -} - -TEST_F(QueueMapSizeTest, AfterPushingOnePoppingOnePerKeyPushingSame) { - push(2, 3); - pop(2); - push(2, 3); - EXPECT_EQ(1, size()); -} - -class QueueMapMemoryLeakTest: public QueueMapTest { -public: - void EXPECT_NUM_INSTANCES(int num) { - EXPECT_EQ(num, MinimalKeyType::instances); - EXPECT_EQ(num, MinimalValueType::instances); - } -}; - -TEST_F(QueueMapMemoryLeakTest, Empty) { - EXPECT_NUM_INSTANCES(0); -} - -TEST_F(QueueMapMemoryLeakTest, AfterPushingOne) { - push(2, 3); - EXPECT_NUM_INSTANCES(1); -} - -TEST_F(QueueMapMemoryLeakTest, AfterPushingTwo) { - push(2, 3); - push(3, 4); - EXPECT_NUM_INSTANCES(2); -} - -TEST_F(QueueMapMemoryLeakTest, AfterPushingTwoAndPoppingOldest) { - push(2, 3); - push(3, 4); - pop(); - EXPECT_NUM_INSTANCES(1); -} - -TEST_F(QueueMapMemoryLeakTest, AfterPushingTwoAndPoppingFirst) { - push(2, 3); - push(3, 4); - pop(2); - EXPECT_NUM_INSTANCES(1); -} - -TEST_F(QueueMapMemoryLeakTest, AfterPushingTwoAndPoppingLast) { - push(2, 3); - push(3, 4); - pop(3); - EXPECT_NUM_INSTANCES(1); -} - -TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOne) { - push(2, 3); - pop(); - EXPECT_NUM_INSTANCES(0); -} - -TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOnePerKey) { - push(2, 3); - pop(2); - EXPECT_NUM_INSTANCES(0); -} - -TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOnePushingOne) { - push(2, 3); - pop(); - push(3, 4); - EXPECT_NUM_INSTANCES(1); -} - -TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOnePerKeyPushingOne) { - push(2, 3); - pop(2); - push(3, 4); - EXPECT_NUM_INSTANCES(1); -} - -TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOnePushingSame) { - push(2, 3); - pop(); - push(2, 3); - EXPECT_NUM_INSTANCES(1); -} - -TEST_F(QueueMapMemoryLeakTest, AfterPushingOnePoppingOnePerKeyPushingSame) { - push(2, 3); - pop(2); - push(2, 3); - EXPECT_NUM_INSTANCES(1); -} - -class QueueMapValueTest: public QueueMapTest {}; - -TEST_F(QueueMapValueTest, PoppingFromEmpty) { - EXPECT_EQ(boost::none, pop()); -} - -TEST_F(QueueMapValueTest, PoppingFromEmptyPerKey) { - EXPECT_EQ(boost::none, pop(2)); -} - -TEST_F(QueueMapValueTest, PoppingNonexistingPerKey) { - push(3, 2); - EXPECT_EQ(boost::none, pop(2)); -} - -TEST_F(QueueMapValueTest, PushingOne) { - push(3, 2); - EXPECT_EQ(2, pop(3).value()); - EXPECT_EQ(boost::none, pop()); -} - -TEST_F(QueueMapValueTest, PushingTwo) { - push(2, 3); - push(3, 4); - EXPECT_EQ(3, pop().value()); - EXPECT_EQ(4, pop().value()); - EXPECT_EQ(boost::none, pop()); -} - -TEST_F(QueueMapValueTest, AfterPushingTwoAndPoppingFirst) { - push(2, 3); - push(3, 4); - pop(2); - EXPECT_EQ(boost::none, pop(2)); - EXPECT_EQ(4, pop(3).value()); - EXPECT_EQ(boost::none, pop()); -} - -TEST_F(QueueMapValueTest, AfterPushingTwoAndPoppingLast) { - push(2, 3); - push(3, 4); - pop(3); - EXPECT_EQ(boost::none, pop(3)); - EXPECT_EQ(3, pop(2).value()); - EXPECT_EQ(boost::none, pop()); -} - -TEST_F(QueueMapValueTest, AfterPushingOnePoppingOne) { - push(2, 3); - pop(); - EXPECT_EQ(boost::none, pop()); - EXPECT_EQ(boost::none, pop(2)); -} - -TEST_F(QueueMapValueTest, AfterPushingOnePoppingOnePerKey) { - push(2, 3); - pop(2); - EXPECT_EQ(boost::none, pop()); - EXPECT_EQ(boost::none, pop(2)); -} - -TEST_F(QueueMapValueTest, AfterPushingOnePoppingOnePushingOne) { - push(2, 3); - pop(); - push(3, 4); - EXPECT_EQ(boost::none, pop(2)); - EXPECT_EQ(4, pop(3).value()); - EXPECT_EQ(boost::none, pop()); -} - -TEST_F(QueueMapValueTest, AfterPushingOnePoppingOnePerKeyPushingOne) { - push(2, 3); - pop(2); - push(3, 4); - EXPECT_EQ(boost::none, pop(2)); - EXPECT_EQ(4, pop(3).value()); - EXPECT_EQ(boost::none, pop()); -} - -TEST_F(QueueMapValueTest, PushingSomePoppingMiddlePerKey) { - push(1, 2); - push(2, 3); - push(3, 4); - push(4, 5); - push(5, 6); - EXPECT_EQ(3, pop(2).value()); - EXPECT_EQ(5, pop(4).value()); - EXPECT_EQ(2, pop().value()); - EXPECT_EQ(4, pop().value()); - EXPECT_EQ(6, pop().value()); - EXPECT_EQ(boost::none, pop()); -} - -TEST_F(QueueMapValueTest, PushingSomePoppingFirstPerKey) { - push(1, 2); - push(2, 3); - push(3, 4); - push(4, 5); - push(5, 6); - EXPECT_EQ(2, pop(1).value()); - EXPECT_EQ(3, pop(2).value()); - EXPECT_EQ(4, pop().value()); - EXPECT_EQ(5, pop().value()); - EXPECT_EQ(6, pop().value()); - EXPECT_EQ(boost::none, pop()); -} - -TEST_F(QueueMapValueTest, PushingSomePoppingLastPerKey) { - push(1, 2); - push(2, 3); - push(3, 4); - push(4, 5); - push(5, 6); - EXPECT_EQ(6, pop(5).value()); - EXPECT_EQ(5, pop(4).value()); - EXPECT_EQ(2, pop().value()); - EXPECT_EQ(3, pop().value()); - EXPECT_EQ(4, pop().value()); - EXPECT_EQ(boost::none, pop()); -} - -//This test forces the underlying datastructure (std::map or std::unordered_map) to grow and reallocate memory. -//So it tests, that QueueMap still works after reallocating memory. -TEST_F(QueueMapValueTest, ManyValues) { - //Push 1 million entries - for (int i = 0; i < 1000000; ++i) { - push(i, 2*i); - } - //pop every other one by key - for (int i = 0; i < 1000000; i += 2) { - EXPECT_EQ(2*i, pop(i).value()); - } - //pop the rest in queue order - for (int i = 1; i < 1000000; i += 2) { - EXPECT_EQ(2*i, peek().value()); - EXPECT_EQ(2*i, pop().value()); - } - EXPECT_EQ(0, size()); - EXPECT_EQ(boost::none, pop()); - EXPECT_EQ(boost::none, peek()); -} - -TEST_F(QueueMapValueTest, PushAlreadyExistingValue) { - push(2, 3); - EXPECT_ANY_THROW( - push(2, 4); - ); -} - -class QueueMapPeekTest: public QueueMapTest {}; - -TEST_F(QueueMapPeekTest, PoppingFromEmpty) { - EXPECT_EQ(boost::none, peek()); -} - -TEST_F(QueueMapPeekTest, PushingOne) { - push(3, 2); - EXPECT_EQ(2, peek().value()); -} - -TEST_F(QueueMapPeekTest, PushingTwo) { - push(2, 3); - push(3, 4); - EXPECT_EQ(3, peek().value()); - EXPECT_EQ(3, peek().value()); - EXPECT_EQ(3, pop().value()); - EXPECT_EQ(4, peek().value()); - EXPECT_EQ(4, peek().value()); - EXPECT_EQ(4, pop().value()); - EXPECT_EQ(boost::none, peek()); - EXPECT_EQ(boost::none, pop()); -} - -TEST_F(QueueMapPeekTest, AfterPushingTwoAndPoppingFirst) { - push(2, 3); - push(3, 4); - pop(2); - EXPECT_EQ(boost::none, pop(2)); - EXPECT_EQ(4, peek().value()); -} - -class CopyableValueType { -public: - static int numCopyConstructorCalled; - CopyableValueType(int value): _value(value) {} - CopyableValueType(const CopyableValueType &rhs): CopyableValueType(rhs._value) { - ++numCopyConstructorCalled; - } - CopyableValueType(CopyableValueType &&rhs): CopyableValueType(rhs._value) { - //Don't increase numCopyConstructorCalled - } - int value() const { - return _value; - } -private: - int _value; -}; -int CopyableValueType::numCopyConstructorCalled = 0; - -//Test that QueueMap uses a move constructor for Value if possible -class QueueMapMoveConstructorTest: public Test { -public: - QueueMapMoveConstructorTest() { - CopyableValueType::numCopyConstructorCalled = 0; - map = make_unique>(); - } - unique_ptr> map; -}; - -TEST_F(QueueMapMoveConstructorTest, PushingAndPopping_MoveIntoMap) { - map->push(MinimalKeyType::create(0), CopyableValueType(2)); - CopyableValueType val = map->pop().value(); - EXPECT_EQ(0, CopyableValueType::numCopyConstructorCalled); -} - -TEST_F(QueueMapMoveConstructorTest, PushingAndPoppingPerKey_MoveIntoMap) { - map->push(MinimalKeyType::create(0), CopyableValueType(2)); - CopyableValueType val = map->pop(MinimalKeyType::create(0)).value(); - EXPECT_EQ(0, CopyableValueType::numCopyConstructorCalled); -} - -TEST_F(QueueMapMoveConstructorTest, PushingAndPopping_CopyIntoMap) { - CopyableValueType value(2); - map->push(MinimalKeyType::create(0), value); - CopyableValueType val = map->pop().value(); - EXPECT_EQ(1, CopyableValueType::numCopyConstructorCalled); -} - -TEST_F(QueueMapMoveConstructorTest, PushingAndPoppingPerKey_CopyIntoMap) { - CopyableValueType value(2); - map->push(MinimalKeyType::create(0), value); - CopyableValueType val = map->pop(MinimalKeyType::create(0)).value(); - EXPECT_EQ(1, CopyableValueType::numCopyConstructorCalled); -} diff --git a/test/implementations/caching/cache/QueueMapTest_MemoryLeak.cpp b/test/implementations/caching/cache/QueueMapTest_MemoryLeak.cpp new file mode 100644 index 00000000..74012fc0 --- /dev/null +++ b/test/implementations/caching/cache/QueueMapTest_MemoryLeak.cpp @@ -0,0 +1,87 @@ +#include "testutils/QueueMapTest.h" + +// Tests that QueueMap calls destructors correctly. +// This is needed, because QueueMap does its own memory management. +class QueueMapTest_MemoryLeak: public QueueMapTest { +public: + void EXPECT_NUM_INSTANCES(int num) { + EXPECT_EQ(num, MinimalKeyType::instances); + EXPECT_EQ(num, MinimalValueType::instances); + } +}; + +TEST_F(QueueMapTest_MemoryLeak, Empty) { + EXPECT_NUM_INSTANCES(0); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOne) { + push(2, 3); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_NUM_INSTANCES(2); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwoAndPoppingOldest) { + push(2, 3); + push(3, 4); + pop(); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingTwoAndPoppingLast) { + push(2, 3); + push(3, 4); + pop(3); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOne) { + push(2, 3); + pop(); + EXPECT_NUM_INSTANCES(0); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePerKey) { + push(2, 3); + pop(2); + EXPECT_NUM_INSTANCES(0); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePushingOne) { + push(2, 3); + pop(); + push(3, 4); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePerKeyPushingOne) { + push(2, 3); + pop(2); + push(3, 4); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePushingSame) { + push(2, 3); + pop(); + push(2, 3); + EXPECT_NUM_INSTANCES(1); +} + +TEST_F(QueueMapTest_MemoryLeak, AfterPushingOnePoppingOnePerKeyPushingSame) { + push(2, 3); + pop(2); + push(2, 3); + EXPECT_NUM_INSTANCES(1); +} diff --git a/test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp b/test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp new file mode 100644 index 00000000..e762f3d0 --- /dev/null +++ b/test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp @@ -0,0 +1,64 @@ +#include +#include +#include "../../../../implementations/caching/cache/QueueMap.h" +#include "testutils/MinimalKeyType.h" + +using namespace blockstore::caching; + +using ::testing::Test; +using std::unique_ptr; +using std::make_unique; + +class CopyableValueType { +public: + static int numCopyConstructorCalled; + CopyableValueType(int value): _value(value) {} + CopyableValueType(const CopyableValueType &rhs): CopyableValueType(rhs._value) { + ++numCopyConstructorCalled; + } + CopyableValueType(CopyableValueType &&rhs): CopyableValueType(rhs._value) { + //Don't increase numCopyConstructorCalled + } + int value() const { + return _value; + } +private: + int _value; +}; +int CopyableValueType::numCopyConstructorCalled = 0; + +//Test that QueueMap uses a move constructor for Value if possible +class QueueMapTest_MoveConstructor: public Test { +public: + QueueMapTest_MoveConstructor() { + CopyableValueType::numCopyConstructorCalled = 0; + map = make_unique>(); + } + unique_ptr> map; +}; + +TEST_F(QueueMapTest_MoveConstructor, PushingAndPopping_MoveIntoMap) { + map->push(MinimalKeyType::create(0), CopyableValueType(2)); + CopyableValueType val = map->pop().value(); + EXPECT_EQ(0, CopyableValueType::numCopyConstructorCalled); +} + +TEST_F(QueueMapTest_MoveConstructor, PushingAndPoppingPerKey_MoveIntoMap) { + map->push(MinimalKeyType::create(0), CopyableValueType(2)); + CopyableValueType val = map->pop(MinimalKeyType::create(0)).value(); + EXPECT_EQ(0, CopyableValueType::numCopyConstructorCalled); +} + +TEST_F(QueueMapTest_MoveConstructor, PushingAndPopping_CopyIntoMap) { + CopyableValueType value(2); + map->push(MinimalKeyType::create(0), value); + CopyableValueType val = map->pop().value(); + EXPECT_EQ(1, CopyableValueType::numCopyConstructorCalled); +} + +TEST_F(QueueMapTest_MoveConstructor, PushingAndPoppingPerKey_CopyIntoMap) { + CopyableValueType value(2); + map->push(MinimalKeyType::create(0), value); + CopyableValueType val = map->pop(MinimalKeyType::create(0)).value(); + EXPECT_EQ(1, CopyableValueType::numCopyConstructorCalled); +} diff --git a/test/implementations/caching/cache/QueueMapTest_Peek.cpp b/test/implementations/caching/cache/QueueMapTest_Peek.cpp new file mode 100644 index 00000000..ba3c7ead --- /dev/null +++ b/test/implementations/caching/cache/QueueMapTest_Peek.cpp @@ -0,0 +1,34 @@ +#include "testutils/QueueMapTest.h" +#include + +class QueueMapPeekTest: public QueueMapTest {}; + +TEST_F(QueueMapPeekTest, PoppingFromEmpty) { + EXPECT_EQ(boost::none, peek()); +} + +TEST_F(QueueMapPeekTest, PushingOne) { + push(3, 2); + EXPECT_EQ(2, peek().value()); +} + +TEST_F(QueueMapPeekTest, PushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_EQ(3, peek().value()); + EXPECT_EQ(3, peek().value()); + EXPECT_EQ(3, pop().value()); + EXPECT_EQ(4, peek().value()); + EXPECT_EQ(4, peek().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(boost::none, peek()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapPeekTest, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, peek().value()); +} diff --git a/test/implementations/caching/cache/QueueMapTest_Size.cpp b/test/implementations/caching/cache/QueueMapTest_Size.cpp new file mode 100644 index 00000000..e4d72c9d --- /dev/null +++ b/test/implementations/caching/cache/QueueMapTest_Size.cpp @@ -0,0 +1,79 @@ +#include "testutils/QueueMapTest.h" + +class QueueMapTest_Size: public QueueMapTest {}; + +TEST_F(QueueMapTest_Size, Empty) { + EXPECT_EQ(0, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOne) { + push(2, 3); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_EQ(2, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingTwoAndPoppingOldest) { + push(2, 3); + push(3, 4); + pop(); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingTwoAndPoppingLast) { + push(2, 3); + push(3, 4); + pop(3); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOne) { + push(2, 3); + pop(); + EXPECT_EQ(0, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePerKey) { + push(2, 3); + pop(2); + EXPECT_EQ(0, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePushingOne) { + push(2, 3); + pop(); + push(3, 4); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePerKeyPushingOne) { + push(2, 3); + pop(2); + push(3, 4); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePushingSame) { + push(2, 3); + pop(); + push(2, 3); + EXPECT_EQ(1, size()); +} + +TEST_F(QueueMapTest_Size, AfterPushingOnePoppingOnePerKeyPushingSame) { + push(2, 3); + pop(2); + push(2, 3); + EXPECT_EQ(1, size()); +} diff --git a/test/implementations/caching/cache/QueueMapTest_Values.cpp b/test/implementations/caching/cache/QueueMapTest_Values.cpp new file mode 100644 index 00000000..1acd5069 --- /dev/null +++ b/test/implementations/caching/cache/QueueMapTest_Values.cpp @@ -0,0 +1,150 @@ +#include "testutils/QueueMapTest.h" + +class QueueMapTest_Values: public QueueMapTest {}; + +TEST_F(QueueMapTest_Values, PoppingFromEmpty) { + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, PoppingFromEmptyPerKey) { + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapTest_Values, PoppingNonexistingPerKey) { + push(3, 2); + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapTest_Values, PushingOne) { + push(3, 2); + EXPECT_EQ(2, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, PushingTwo) { + push(2, 3); + push(3, 4); + EXPECT_EQ(3, pop().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, AfterPushingTwoAndPoppingFirst) { + push(2, 3); + push(3, 4); + pop(2); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, AfterPushingTwoAndPoppingLast) { + push(2, 3); + push(3, 4); + pop(3); + EXPECT_EQ(boost::none, pop(3)); + EXPECT_EQ(3, pop(2).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOne) { + push(2, 3); + pop(); + EXPECT_EQ(boost::none, pop()); + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOnePerKey) { + push(2, 3); + pop(2); + EXPECT_EQ(boost::none, pop()); + EXPECT_EQ(boost::none, pop(2)); +} + +TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOnePushingOne) { + push(2, 3); + pop(); + push(3, 4); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, AfterPushingOnePoppingOnePerKeyPushingOne) { + push(2, 3); + pop(2); + push(3, 4); + EXPECT_EQ(boost::none, pop(2)); + EXPECT_EQ(4, pop(3).value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, PushingSomePoppingMiddlePerKey) { + push(1, 2); + push(2, 3); + push(3, 4); + push(4, 5); + push(5, 6); + EXPECT_EQ(3, pop(2).value()); + EXPECT_EQ(5, pop(4).value()); + EXPECT_EQ(2, pop().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(6, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, PushingSomePoppingFirstPerKey) { + push(1, 2); + push(2, 3); + push(3, 4); + push(4, 5); + push(5, 6); + EXPECT_EQ(2, pop(1).value()); + EXPECT_EQ(3, pop(2).value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(5, pop().value()); + EXPECT_EQ(6, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +TEST_F(QueueMapTest_Values, PushingSomePoppingLastPerKey) { + push(1, 2); + push(2, 3); + push(3, 4); + push(4, 5); + push(5, 6); + EXPECT_EQ(6, pop(5).value()); + EXPECT_EQ(5, pop(4).value()); + EXPECT_EQ(2, pop().value()); + EXPECT_EQ(3, pop().value()); + EXPECT_EQ(4, pop().value()); + EXPECT_EQ(boost::none, pop()); +} + +//This test forces the underlying datastructure (std::map or std::unordered_map) to grow and reallocate memory. +//So it tests, that QueueMap still works after reallocating memory. +TEST_F(QueueMapTest_Values, ManyValues) { + //Push 1 million entries + for (int i = 0; i < 1000000; ++i) { + push(i, 2*i); + } + //pop every other one by key + for (int i = 0; i < 1000000; i += 2) { + EXPECT_EQ(2*i, pop(i).value()); + } + //pop the rest in queue order + for (int i = 1; i < 1000000; i += 2) { + EXPECT_EQ(2*i, peek().value()); + EXPECT_EQ(2*i, pop().value()); + } + EXPECT_EQ(0, size()); + EXPECT_EQ(boost::none, pop()); + EXPECT_EQ(boost::none, peek()); +} + +TEST_F(QueueMapTest_Values, PushAlreadyExistingValue) { + push(2, 3); + EXPECT_ANY_THROW( + push(2, 4); + ); +} diff --git a/test/implementations/caching/cache/testutils/MinimalKeyType.cpp b/test/implementations/caching/cache/testutils/MinimalKeyType.cpp new file mode 100644 index 00000000..f6b327ff --- /dev/null +++ b/test/implementations/caching/cache/testutils/MinimalKeyType.cpp @@ -0,0 +1,3 @@ +#include "MinimalKeyType.h" + +int MinimalKeyType::instances = 0; diff --git a/test/implementations/caching/cache/testutils/MinimalKeyType.h b/test/implementations/caching/cache/testutils/MinimalKeyType.h new file mode 100644 index 00000000..9133d1f8 --- /dev/null +++ b/test/implementations/caching/cache/testutils/MinimalKeyType.h @@ -0,0 +1,47 @@ +#pragma once +#ifndef BLOCKS_MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALKEYTYPE_H_ +#define BLOCKS_MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALKEYTYPE_H_ + +#include + +// This is a not-default-constructible Key type +class MinimalKeyType { +public: + static int instances; + + static MinimalKeyType create(int value) { + return MinimalKeyType(value); + } + + MinimalKeyType(const MinimalKeyType &rhs): MinimalKeyType(rhs.value()) { + } + + ~MinimalKeyType() { + --instances; + } + + int value() const { + return _value; + } + +private: + MinimalKeyType(int value): _value(value) { + ++instances; + } + + int _value; +}; + +namespace std { +template <> struct hash { + size_t operator()(const MinimalKeyType &obj) const { + return obj.value(); + } +}; +} + +inline bool operator==(const MinimalKeyType &lhs, const MinimalKeyType &rhs) { + return lhs.value() == rhs.value(); +} + +#endif diff --git a/test/implementations/caching/cache/testutils/MinimalValueType.cpp b/test/implementations/caching/cache/testutils/MinimalValueType.cpp new file mode 100644 index 00000000..f5cbdbae --- /dev/null +++ b/test/implementations/caching/cache/testutils/MinimalValueType.cpp @@ -0,0 +1,3 @@ +#include "MinimalValueType.h" + +int MinimalValueType::instances = 0; diff --git a/test/implementations/caching/cache/testutils/MinimalValueType.h b/test/implementations/caching/cache/testutils/MinimalValueType.h new file mode 100644 index 00000000..9e48e43e --- /dev/null +++ b/test/implementations/caching/cache/testutils/MinimalValueType.h @@ -0,0 +1,44 @@ +#pragma once +#ifndef BLOCKS_MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALVALUETYPE_H_ +#define BLOCKS_MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_MINIMALVALUETYPE_H_ + +#include +#include + +// This is a not-default-constructible non-copyable but moveable Value type +class MinimalValueType { +public: + static int instances; + + static MinimalValueType create(int value) { + return MinimalValueType(value); + } + + MinimalValueType(MinimalValueType &&rhs): MinimalValueType(rhs.value()) { + rhs._isMoved = true; + } + + ~MinimalValueType() { + assert(!_isDestructed); + --instances; + _isDestructed = true; + } + + int value() const { + assert(!_isMoved && !_isDestructed); + return _value; + } + +private: + MinimalValueType(int value): _value(value), _isMoved(false), _isDestructed(false) { + ++instances; + } + + int _value; + bool _isMoved; + bool _isDestructed; + + DISALLOW_COPY_AND_ASSIGN(MinimalValueType); +}; + +#endif diff --git a/test/implementations/caching/cache/testutils/QueueMapTest.cpp b/test/implementations/caching/cache/testutils/QueueMapTest.cpp new file mode 100644 index 00000000..f61b41a3 --- /dev/null +++ b/test/implementations/caching/cache/testutils/QueueMapTest.cpp @@ -0,0 +1,45 @@ +#include "QueueMapTest.h" + +QueueMapTest::QueueMapTest() { + MinimalKeyType::instances = 0; + MinimalValueType::instances = 0; + _map = std::make_unique>(); +} + +QueueMapTest::~QueueMapTest() { + _map.reset(); + EXPECT_EQ(0, MinimalKeyType::instances); + EXPECT_EQ(0, MinimalValueType::instances); +} + +void QueueMapTest::push(int key, int value) { + _map->push(MinimalKeyType::create(key), MinimalValueType::create(value)); +} + +boost::optional QueueMapTest::pop() { + auto elem = _map->pop(); + if (!elem) { + return boost::none; + } + return elem.value().value(); +} + +boost::optional QueueMapTest::pop(int key) { + auto elem = _map->pop(MinimalKeyType::create(key)); + if (!elem) { + return boost::none; + } + return elem.value().value(); +} + +boost::optional QueueMapTest::peek() { + auto elem = _map->peek(); + if (!elem) { + return boost::none; + } + return elem.value().value(); +} + +int QueueMapTest::size() { + return _map->size(); +} diff --git a/test/implementations/caching/cache/testutils/QueueMapTest.h b/test/implementations/caching/cache/testutils/QueueMapTest.h new file mode 100644 index 00000000..e49ad93f --- /dev/null +++ b/test/implementations/caching/cache/testutils/QueueMapTest.h @@ -0,0 +1,32 @@ +#pragma once +#ifndef BLOCKS_MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_ +#define BLOCKS_MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_ + +#include +#include +#include "../../../../../implementations/caching/cache/QueueMap.h" +#include "MinimalKeyType.h" +#include "MinimalValueType.h" +#include + +// This class is a parent class for tests on QueueMap. +// It offers functions to work with a QueueMap test object which is built using types having only the minimal type requirements. +// Furthermore, the class checks that there are no memory leaks left after destructing the QueueMap (by counting leftover instances of Keys/Values). +class QueueMapTest: public ::testing::Test { +public: + QueueMapTest(); + ~QueueMapTest(); + + void push(int key, int value); + boost::optional pop(); + boost::optional pop(int key); + boost::optional peek(); + int size(); + +private: + std::unique_ptr> _map; +}; + + + +#endif From d589910b0dd0c3ba93d85e05b706c438f1c6f42c Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 27 Apr 2015 23:10:30 +0200 Subject: [PATCH 30/41] TODOs --- implementations/caching/cache/Cache.h | 2 -- test/implementations/caching/cache/CacheTest.cpp | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/implementations/caching/cache/Cache.h b/implementations/caching/cache/Cache.h index f47e6c95..951c8f66 100644 --- a/implementations/caching/cache/Cache.h +++ b/implementations/caching/cache/Cache.h @@ -12,8 +12,6 @@ namespace blockstore { namespace caching { -//TODO Test - template class Cache { public: diff --git a/test/implementations/caching/cache/CacheTest.cpp b/test/implementations/caching/cache/CacheTest.cpp index e746baaa..2305fdf3 100644 --- a/test/implementations/caching/cache/CacheTest.cpp +++ b/test/implementations/caching/cache/CacheTest.cpp @@ -10,3 +10,5 @@ class CacheTest: public Test { public: Cache cache; }; + +//TODO Write test cases From cb402fd14b0450964dffc2afed5765c428bffb23 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Tue, 28 Apr 2015 11:56:07 +0200 Subject: [PATCH 31/41] Added test cases for Cache --- .../caching/cache/CacheTest.cpp | 14 -- .../cache/CacheTest_MoveConstructor.cpp | 34 +++++ .../caching/cache/CacheTest_PushAndPop.cpp | 133 ++++++++++++++++++ .../cache/QueueMapTest_MoveConstructor.cpp | 49 +++---- .../caching/cache/testutils/CacheTest.cpp | 13 ++ .../caching/cache/testutils/CacheTest.h | 25 ++++ .../testutils/CopyableMovableValueType.cpp | 3 + .../testutils/CopyableMovableValueType.h | 23 +++ 8 files changed, 247 insertions(+), 47 deletions(-) delete mode 100644 test/implementations/caching/cache/CacheTest.cpp create mode 100644 test/implementations/caching/cache/CacheTest_MoveConstructor.cpp create mode 100644 test/implementations/caching/cache/CacheTest_PushAndPop.cpp create mode 100644 test/implementations/caching/cache/testutils/CacheTest.cpp create mode 100644 test/implementations/caching/cache/testutils/CacheTest.h create mode 100644 test/implementations/caching/cache/testutils/CopyableMovableValueType.cpp create mode 100644 test/implementations/caching/cache/testutils/CopyableMovableValueType.h diff --git a/test/implementations/caching/cache/CacheTest.cpp b/test/implementations/caching/cache/CacheTest.cpp deleted file mode 100644 index 2305fdf3..00000000 --- a/test/implementations/caching/cache/CacheTest.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include - -#include "../../../../implementations/caching/cache/Cache.h" - -using ::testing::Test; - -using namespace blockstore::caching; - -class CacheTest: public Test { -public: - Cache cache; -}; - -//TODO Write test cases diff --git a/test/implementations/caching/cache/CacheTest_MoveConstructor.cpp b/test/implementations/caching/cache/CacheTest_MoveConstructor.cpp new file mode 100644 index 00000000..925ab99f --- /dev/null +++ b/test/implementations/caching/cache/CacheTest_MoveConstructor.cpp @@ -0,0 +1,34 @@ +#include +#include +#include "../../../../implementations/caching/cache/Cache.h" +#include "testutils/MinimalKeyType.h" +#include "testutils/CopyableMovableValueType.h" + +using namespace blockstore::caching; + +using ::testing::Test; +using std::unique_ptr; +using std::make_unique; + +//Test that Cache uses a move constructor for Value if possible +class CacheTest_MoveConstructor: public Test { +public: + CacheTest_MoveConstructor() { + CopyableMovableValueType::numCopyConstructorCalled = 0; + cache = make_unique>(); + } + unique_ptr> cache; +}; + +TEST_F(CacheTest_MoveConstructor, MoveIntoCache) { + cache->push(MinimalKeyType::create(0), CopyableMovableValueType(2)); + CopyableMovableValueType val = cache->pop(MinimalKeyType::create(0)).value(); + EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled); +} + +TEST_F(CacheTest_MoveConstructor, CopyIntoCache) { + CopyableMovableValueType value(2); + cache->push(MinimalKeyType::create(0), value); + CopyableMovableValueType val = cache->pop(MinimalKeyType::create(0)).value(); + EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled); +} diff --git a/test/implementations/caching/cache/CacheTest_PushAndPop.cpp b/test/implementations/caching/cache/CacheTest_PushAndPop.cpp new file mode 100644 index 00000000..6203fe1a --- /dev/null +++ b/test/implementations/caching/cache/CacheTest_PushAndPop.cpp @@ -0,0 +1,133 @@ +#include "testutils/CacheTest.h" + +#include "../../../../implementations/caching/cache/Cache.h" +#include "testutils/MinimalKeyType.h" +#include "testutils/MinimalValueType.h" + +using ::testing::Test; + +using namespace blockstore::caching; + +class CacheTest_PushAndPop: public CacheTest {}; + +TEST_F(CacheTest_PushAndPop, PopNonExistingEntry_EmptyCache) { + EXPECT_EQ(boost::none, pop(10)); +} + +TEST_F(CacheTest_PushAndPop, PopNonExistingEntry_NonEmptyCache) { + push(9, 10); + EXPECT_EQ(boost::none, pop(10)); +} + +TEST_F(CacheTest_PushAndPop, PopNonExistingEntry_FullCache) { + //Add a lot of even numbered keys + for (int i = 0; i < Cache::MAX_ENTRIES; ++i) { + push(2*i, 2*i); + } + //Request an odd numbered key + EXPECT_EQ(boost::none, pop(9)); +} + +TEST_F(CacheTest_PushAndPop, OneEntry) { + push(10, 20); + EXPECT_EQ(20, pop(10).value()); +} + +TEST_F(CacheTest_PushAndPop, MultipleEntries) { + push(10, 20); + push(20, 30); + push(30, 40); + EXPECT_EQ(30, pop(20).value()); + EXPECT_EQ(20, pop(10).value()); + EXPECT_EQ(40, pop(30).value()); +} + +TEST_F(CacheTest_PushAndPop, FullCache) { + for(int i = 0; i < Cache::MAX_ENTRIES; ++i) { + push(i, 2*i); + } + for(int i = 0; i < Cache::MAX_ENTRIES; ++i) { + EXPECT_EQ(2*i, pop(i).value()); + } +} + +TEST_F(CacheTest_PushAndPop, FullCache_PushNonOrdered_PopOrdered) { + for(int i = 1; i < Cache::MAX_ENTRIES; i += 2) { + push(i, 2*i); + } + for(int i = 0; i < Cache::MAX_ENTRIES; i += 2) { + push(i, 2*i); + } + for(int i = 0; i < Cache::MAX_ENTRIES; ++i) { + EXPECT_EQ(2*i, pop(i).value()); + } +} + +TEST_F(CacheTest_PushAndPop, FullCache_PushOrdered_PopNonOrdered) { + for(int i = 0; i < Cache::MAX_ENTRIES; ++i) { + push(i, 2*i); + } + for(int i = 1; i < Cache::MAX_ENTRIES; i += 2) { + EXPECT_EQ(2*i, pop(i).value()); + } + for(int i = 0; i < Cache::MAX_ENTRIES; i += 2) { + EXPECT_EQ(2*i, pop(i).value()); + } +} + +int roundDownToEven(int number) { + if (number % 2 == 0) { + return number; + } else { + return number - 1; + } +} + +int roundDownToOdd(int number) { + if (number % 2 != 0) { + return number; + } else { + return number - 1; + } +} + +TEST_F(CacheTest_PushAndPop, FullCache_PushNonOrdered_PopNonOrdered) { + for(int i = roundDownToEven(Cache::MAX_ENTRIES - 1); i >= 0; i -= 2) { + push(i, 2*i); + } + for(int i = 1; i < Cache::MAX_ENTRIES; i += 2) { + push(i, 2*i); + } + for(int i = roundDownToOdd(Cache::MAX_ENTRIES-1); i >= 0; i -= 2) { + EXPECT_EQ(2*i, pop(i).value()); + } + for(int i = 0; i < Cache::MAX_ENTRIES; i += 2) { + EXPECT_EQ(2*i, pop(i).value()); + } +} + +TEST_F(CacheTest_PushAndPop, MoreThanFullCache) { + for(int i = 0; i < Cache::MAX_ENTRIES + 2; ++i) { + push(i, 2*i); + } + //Check that the oldest two elements got deleted automatically + EXPECT_EQ(boost::none, pop(0)); + EXPECT_EQ(boost::none, pop(1)); + //Check the other elements are still there + for(int i = 2; i < Cache::MAX_ENTRIES + 2; ++i) { + EXPECT_EQ(2*i, pop(i).value()); + } +} + +TEST_F(CacheTest_PushAndPop, AfterTimeout) { + constexpr double TIMEOUT1_SEC = Cache::MAX_LIFETIME_SEC * 3/4; + constexpr double TIMEOUT2_SEC = Cache::PURGE_LIFETIME_SEC * 3/4; + static_assert(TIMEOUT1_SEC + TIMEOUT2_SEC > Cache::MAX_LIFETIME_SEC, "Ensure that our chosen timeouts push the first entry out of the cache"); + + push(10, 20); + boost::this_thread::sleep_for(boost::chrono::milliseconds(static_cast(1000 * TIMEOUT1_SEC))); + push(20, 30); + boost::this_thread::sleep_for(boost::chrono::milliseconds(static_cast(1000 * TIMEOUT2_SEC))); + EXPECT_EQ(boost::none, pop(10)); + EXPECT_EQ(30, pop(20).value()); +} diff --git a/test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp b/test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp index e762f3d0..c0bebd71 100644 --- a/test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp +++ b/test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp @@ -2,6 +2,7 @@ #include #include "../../../../implementations/caching/cache/QueueMap.h" #include "testutils/MinimalKeyType.h" +#include "testutils/CopyableMovableValueType.h" using namespace blockstore::caching; @@ -9,56 +10,38 @@ using ::testing::Test; using std::unique_ptr; using std::make_unique; -class CopyableValueType { -public: - static int numCopyConstructorCalled; - CopyableValueType(int value): _value(value) {} - CopyableValueType(const CopyableValueType &rhs): CopyableValueType(rhs._value) { - ++numCopyConstructorCalled; - } - CopyableValueType(CopyableValueType &&rhs): CopyableValueType(rhs._value) { - //Don't increase numCopyConstructorCalled - } - int value() const { - return _value; - } -private: - int _value; -}; -int CopyableValueType::numCopyConstructorCalled = 0; - //Test that QueueMap uses a move constructor for Value if possible class QueueMapTest_MoveConstructor: public Test { public: QueueMapTest_MoveConstructor() { - CopyableValueType::numCopyConstructorCalled = 0; - map = make_unique>(); + CopyableMovableValueType::numCopyConstructorCalled = 0; + map = make_unique>(); } - unique_ptr> map; + unique_ptr> map; }; TEST_F(QueueMapTest_MoveConstructor, PushingAndPopping_MoveIntoMap) { - map->push(MinimalKeyType::create(0), CopyableValueType(2)); - CopyableValueType val = map->pop().value(); - EXPECT_EQ(0, CopyableValueType::numCopyConstructorCalled); + map->push(MinimalKeyType::create(0), CopyableMovableValueType(2)); + CopyableMovableValueType val = map->pop().value(); + EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled); } TEST_F(QueueMapTest_MoveConstructor, PushingAndPoppingPerKey_MoveIntoMap) { - map->push(MinimalKeyType::create(0), CopyableValueType(2)); - CopyableValueType val = map->pop(MinimalKeyType::create(0)).value(); - EXPECT_EQ(0, CopyableValueType::numCopyConstructorCalled); + map->push(MinimalKeyType::create(0), CopyableMovableValueType(2)); + CopyableMovableValueType val = map->pop(MinimalKeyType::create(0)).value(); + EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled); } TEST_F(QueueMapTest_MoveConstructor, PushingAndPopping_CopyIntoMap) { - CopyableValueType value(2); + CopyableMovableValueType value(2); map->push(MinimalKeyType::create(0), value); - CopyableValueType val = map->pop().value(); - EXPECT_EQ(1, CopyableValueType::numCopyConstructorCalled); + CopyableMovableValueType val = map->pop().value(); + EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled); } TEST_F(QueueMapTest_MoveConstructor, PushingAndPoppingPerKey_CopyIntoMap) { - CopyableValueType value(2); + CopyableMovableValueType value(2); map->push(MinimalKeyType::create(0), value); - CopyableValueType val = map->pop(MinimalKeyType::create(0)).value(); - EXPECT_EQ(1, CopyableValueType::numCopyConstructorCalled); + CopyableMovableValueType val = map->pop(MinimalKeyType::create(0)).value(); + EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled); } diff --git a/test/implementations/caching/cache/testutils/CacheTest.cpp b/test/implementations/caching/cache/testutils/CacheTest.cpp new file mode 100644 index 00000000..b93bbccb --- /dev/null +++ b/test/implementations/caching/cache/testutils/CacheTest.cpp @@ -0,0 +1,13 @@ +#include "CacheTest.h" + +void CacheTest::push(int key, int value) { + return _cache.push(MinimalKeyType::create(key), MinimalValueType::create(value)); +} + +boost::optional CacheTest::pop(int key) { + boost::optional entry = _cache.pop(MinimalKeyType::create(key)); + if (!entry) { + return boost::none; + } + return entry->value(); +} diff --git a/test/implementations/caching/cache/testutils/CacheTest.h b/test/implementations/caching/cache/testutils/CacheTest.h new file mode 100644 index 00000000..61769936 --- /dev/null +++ b/test/implementations/caching/cache/testutils/CacheTest.h @@ -0,0 +1,25 @@ +#pragma once +#ifndef BLOCKS_MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_ +#define BLOCKS_MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_QUEUEMAPTEST_H_ + +#include +#include "../../../../../implementations/caching/cache/Cache.h" +#include "MinimalKeyType.h" +#include "MinimalValueType.h" +#include + +// This class is a parent class for tests on QueueMap. +// It offers functions to work with a QueueMap test object which is built using types having only the minimal type requirements. +// Furthermore, the class checks that there are no memory leaks left after destructing the QueueMap (by counting leftover instances of Keys/Values). +class CacheTest: public ::testing::Test { +public: + void push(int key, int value); + boost::optional pop(int key); + + using Cache = blockstore::caching::Cache; + +private: + Cache _cache; +}; + +#endif diff --git a/test/implementations/caching/cache/testutils/CopyableMovableValueType.cpp b/test/implementations/caching/cache/testutils/CopyableMovableValueType.cpp new file mode 100644 index 00000000..cf23bd88 --- /dev/null +++ b/test/implementations/caching/cache/testutils/CopyableMovableValueType.cpp @@ -0,0 +1,3 @@ +#include "CopyableMovableValueType.h" + +int CopyableMovableValueType::numCopyConstructorCalled = 0; diff --git a/test/implementations/caching/cache/testutils/CopyableMovableValueType.h b/test/implementations/caching/cache/testutils/CopyableMovableValueType.h new file mode 100644 index 00000000..c3201423 --- /dev/null +++ b/test/implementations/caching/cache/testutils/CopyableMovableValueType.h @@ -0,0 +1,23 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_COPYABLEMOVABLEVALUETYPE_H_ +#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_CACHING_CACHE_TESTUTILS_COPYABLEMOVABLEVALUETYPE_H_ + +class CopyableMovableValueType { +public: + static int numCopyConstructorCalled; + CopyableMovableValueType(int value): _value(value) {} + CopyableMovableValueType(const CopyableMovableValueType &rhs): CopyableMovableValueType(rhs._value) { + ++numCopyConstructorCalled; + } + CopyableMovableValueType(CopyableMovableValueType &&rhs): CopyableMovableValueType(rhs._value) { + //Don't increase numCopyConstructorCalled + } + int value() const { + return _value; + } +private: + int _value; +}; + + +#endif From 5802040d7882146e9b37a7b0887d10efa158fd36 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Tue, 28 Apr 2015 13:19:44 +0200 Subject: [PATCH 32/41] Removed unused exception class --- .../ondisk/FileAlreadyExistsException.cpp | 19 ---------------- .../ondisk/FileAlreadyExistsException.h | 22 ------------------- implementations/ondisk/OnDiskBlock.cpp | 1 - 3 files changed, 42 deletions(-) delete mode 100644 implementations/ondisk/FileAlreadyExistsException.cpp delete mode 100644 implementations/ondisk/FileAlreadyExistsException.h diff --git a/implementations/ondisk/FileAlreadyExistsException.cpp b/implementations/ondisk/FileAlreadyExistsException.cpp deleted file mode 100644 index 904b138c..00000000 --- a/implementations/ondisk/FileAlreadyExistsException.cpp +++ /dev/null @@ -1,19 +0,0 @@ -#include "FileAlreadyExistsException.h" - -namespace bf = boost::filesystem; - -using std::runtime_error; -using std::string; - -namespace blockstore { -namespace ondisk { - -FileAlreadyExistsException::FileAlreadyExistsException(const bf::path &filepath) -: runtime_error(string("The file ")+filepath.c_str()+" already exists") { -} - -FileAlreadyExistsException::~FileAlreadyExistsException() { -} - -} -} diff --git a/implementations/ondisk/FileAlreadyExistsException.h b/implementations/ondisk/FileAlreadyExistsException.h deleted file mode 100644 index 85cf18e5..00000000 --- a/implementations/ondisk/FileAlreadyExistsException.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once -#ifndef BLOCKSTORE_IMPLEMENTATIONS_ONDISK_FILEALREADYEXISTSEXCEPTION_H_ -#define BLOCKSTORE_IMPLEMENTATIONS_ONDISK_FILEALREADYEXISTSEXCEPTION_H_ - -#include - -#include - -namespace blockstore { -namespace ondisk { - -//TODO We probably don't want an exception for that -class FileAlreadyExistsException: public std::runtime_error { -public: - explicit FileAlreadyExistsException(const boost::filesystem::path &filepath); - virtual ~FileAlreadyExistsException(); -}; - -} -} - -#endif diff --git a/implementations/ondisk/OnDiskBlock.cpp b/implementations/ondisk/OnDiskBlock.cpp index 6fb621d2..4139a999 100644 --- a/implementations/ondisk/OnDiskBlock.cpp +++ b/implementations/ondisk/OnDiskBlock.cpp @@ -1,7 +1,6 @@ #include #include #include -#include "FileAlreadyExistsException.h" #include "OnDiskBlock.h" #include "OnDiskBlockStore.h" #include "../../utils/FileDoesntExistException.h" From fd09ff1f9e4214e6ace6a203383395a2f26f9dd8 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Wed, 29 Apr 2015 16:33:26 +0200 Subject: [PATCH 33/41] Remove includes to unused exception --- .../ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp | 1 - .../ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp index 5f1fe07d..89fe0bfa 100644 --- a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp +++ b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockCreateTest.cpp @@ -1,4 +1,3 @@ -#include "../../../../implementations/ondisk/FileAlreadyExistsException.h" #include "../../../../implementations/ondisk/OnDiskBlock.h" #include "google/gtest/gtest.h" diff --git a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp index f0638639..8d723b92 100644 --- a/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp +++ b/test/implementations/ondisk/OnDiskBlockTest/OnDiskBlockFlushTest.cpp @@ -1,4 +1,3 @@ -#include "../../../../implementations/ondisk/FileAlreadyExistsException.h" #include "../../../../implementations/ondisk/OnDiskBlock.h" #include #include "google/gtest/gtest.h" From 8e9372ffafbf6745d4a5638549aeab1d3cf6f720 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 4 May 2015 20:20:16 +0200 Subject: [PATCH 34/41] Add appveyor.yml --- appveyor.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..7da3d0d9 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,35 @@ +version: 0.9.{build} + +install: + - ps: wget https://s3.amazonaws.com/biibinaries/thirdparty/cmake-3.0.2-win32-x86.zip -OutFile cmake.zip + - cmd: echo "Unzipping cmake..." + - cmd: 7z x cmake.zip -o"C:\Program Files (x86)\" -y > nul + - cmd: set PATH=%PATH:CMake 2.8\bin=%;C:\Program Files (x86)\cmake-3.0.2-win32-x86\bin + - cmd: cmake --version + - cmd: echo "Downloading biicode..." + - ps: wget http://www.biicode.com/downloads/latest/win -OutFile bii-win.exe + - cmd: bii-win.exe /VERYSILENT + - cmd: set PATH=%PATH%;C:\Program Files (x86)\BiiCode\bii + - cmd: bii -v + - cmd: del bii-win.exe + - cmd: del cmake.zip + +before_build: + - cmd: bii init -L + # copy files and folders + - cmd: xcopy "%APPVEYOR_BUILD_FOLDER%" blocks\%block_user%\%block_name%\ /e + - cmd: bii configure -G "Visual Studio 12" + +build_script: + - cmd: bii build + +test_script: + - cmd: cd bin + - cmd: amalulla_cpp-expression-parser_test-shunting-yard.exe + +environment: + block_user: + "messmer" + block_name: + "blockstore" + From dbbf9db80eff11a16bcffd31358d81416e178075 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 4 May 2015 21:42:56 +0200 Subject: [PATCH 35/41] Add debug output --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 7da3d0d9..6ab16fd7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,7 +17,8 @@ install: before_build: - cmd: bii init -L # copy files and folders - - cmd: xcopy "%APPVEYOR_BUILD_FOLDER%" blocks\%block_user%\%block_name%\ /e + - cmd: dir + - cmd: echo xcopy "%APPVEYOR_BUILD_FOLDER%" blocks\%block_user%\%block_name%\ /e - cmd: bii configure -G "Visual Studio 12" build_script: From 1e05ab698c357b3e79443943ba54dcf67b7ee746 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 4 May 2015 22:01:41 +0200 Subject: [PATCH 36/41] Fix appveyor? --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 6ab16fd7..15fe75c4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,5 @@ version: 0.9.{build} +os: Visual Studio 2015 RC install: - ps: wget https://s3.amazonaws.com/biibinaries/thirdparty/cmake-3.0.2-win32-x86.zip -OutFile cmake.zip @@ -18,7 +19,7 @@ before_build: - cmd: bii init -L # copy files and folders - cmd: dir - - cmd: echo xcopy "%APPVEYOR_BUILD_FOLDER%" blocks\%block_user%\%block_name%\ /e +# - cmd: echo xcopy "%APPVEYOR_BUILD_FOLDER%" blocks\%block_user%\%block_name%\ /e - cmd: bii configure -G "Visual Studio 12" build_script: From a36ab8e2d79911abde23a013127e03714ef2fa83 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Mon, 4 May 2015 22:13:50 +0200 Subject: [PATCH 37/41] Fix appveyor? --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 15fe75c4..cff5981a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,7 +2,7 @@ version: 0.9.{build} os: Visual Studio 2015 RC install: - - ps: wget https://s3.amazonaws.com/biibinaries/thirdparty/cmake-3.0.2-win32-x86.zip -OutFile cmake.zip + - ps: wget http://www.cmake.org/files/v3.2/cmake-3.2.2-win32-x86.zip -OutFile cmake.zip - cmd: echo "Unzipping cmake..." - cmd: 7z x cmake.zip -o"C:\Program Files (x86)\" -y > nul - cmd: set PATH=%PATH:CMake 2.8\bin=%;C:\Program Files (x86)\cmake-3.0.2-win32-x86\bin @@ -20,7 +20,7 @@ before_build: # copy files and folders - cmd: dir # - cmd: echo xcopy "%APPVEYOR_BUILD_FOLDER%" blocks\%block_user%\%block_name%\ /e - - cmd: bii configure -G "Visual Studio 12" + - cmd: bii configure -G "Visual Studio 14 2015" build_script: - cmd: bii build From 0042ae1cef516a598dc7245ca5d4718302cf641c Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Wed, 6 May 2015 00:09:11 +0200 Subject: [PATCH 38/41] - Run test cases for EncryptedBlockStore with different ciphers - Implement FakeAuthenticatedCipher for use with specific EncryptedBlockStoreTest - Write skeleton for specific EncryptedBlockStoreTest - Fix behavior of AES256_CFB when called with too small input - Add testcase that all ciphers (also non-authenticating ones) have to handle too small input correctly --- .../encrypted/ciphers/AES256_CFB.cpp | 4 + implementations/encrypted/ciphers/Cipher.h | 5 +- test/implementations/encrypted/CipherTest.cpp | 65 +++++------ .../encrypted/EncryptedBlockStoreTest.cpp | 28 ----- .../EncryptedBlockStoreTest_Generic.cpp | 40 +++++++ .../EncryptedBlockStoreTest_Specific.cpp | 35 ++++++ .../testutils/FakeAuthenticatedCipher.cpp | 3 + .../testutils/FakeAuthenticatedCipher.h | 102 ++++++++++++++++++ 8 files changed, 221 insertions(+), 61 deletions(-) delete mode 100644 test/implementations/encrypted/EncryptedBlockStoreTest.cpp create mode 100644 test/implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp create mode 100644 test/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp create mode 100644 test/implementations/encrypted/testutils/FakeAuthenticatedCipher.cpp create mode 100644 test/implementations/encrypted/testutils/FakeAuthenticatedCipher.h diff --git a/implementations/encrypted/ciphers/AES256_CFB.cpp b/implementations/encrypted/ciphers/AES256_CFB.cpp index 63d93ac7..c7737b1b 100644 --- a/implementations/encrypted/ciphers/AES256_CFB.cpp +++ b/implementations/encrypted/ciphers/AES256_CFB.cpp @@ -21,6 +21,10 @@ Data AES256_CFB::encrypt(const byte *plaintext, unsigned int plaintextSize, cons } boost::optional AES256_CFB::decrypt(const byte *ciphertext, unsigned int ciphertextSize, const EncryptionKey &encKey) { + if (ciphertextSize < IV_SIZE) { + return boost::none; + } + const byte *ciphertextIV = ciphertext; const byte *ciphertextData = ciphertext + IV_SIZE; auto decryption = CFB_Mode::Decryption((byte*)encKey.data(), encKey.BINARY_LENGTH, ciphertextIV); diff --git a/implementations/encrypted/ciphers/Cipher.h b/implementations/encrypted/ciphers/Cipher.h index 5dddf4fc..5195bcb4 100644 --- a/implementations/encrypted/ciphers/Cipher.h +++ b/implementations/encrypted/ciphers/Cipher.h @@ -4,6 +4,7 @@ #include #include +#include namespace blockstore { namespace encrypted { @@ -15,8 +16,8 @@ public: same_type(UINT32_C(0), X::ciphertextSize(UINT32_C(5))); same_type(UINT32_C(0), X::plaintextSize(UINT32_C(5))); typename X::EncryptionKey key = X::EncryptionKey::CreateRandom(); - same_type(cpputils::Data(0), X::encrypt((byte*)nullptr, UINT32_C(0), key)); - same_type(boost::optional(cpputils::Data(0)), X::decrypt((byte*)nullptr, UINT32_C(0), key)); + same_type(cpputils::Data(0), X::encrypt((uint8_t*)nullptr, UINT32_C(0), key)); + same_type(boost::optional(cpputils::Data(0)), X::decrypt((uint8_t*)nullptr, UINT32_C(0), key)); } private: diff --git a/test/implementations/encrypted/CipherTest.cpp b/test/implementations/encrypted/CipherTest.cpp index 609a2717..f5581b05 100644 --- a/test/implementations/encrypted/CipherTest.cpp +++ b/test/implementations/encrypted/CipherTest.cpp @@ -1,7 +1,8 @@ #include +#include "../../../implementations/encrypted/ciphers/Cipher.h" #include "../../../implementations/encrypted/ciphers/AES256_CFB.h" #include "../../../implementations/encrypted/ciphers/AES256_GCM.h" -#include "../../../implementations/encrypted/ciphers/Cipher.h" +#include "testutils/FakeAuthenticatedCipher.h" #include #include @@ -16,9 +17,9 @@ template class CipherTest: public ::testing::Test { public: BOOST_CONCEPT_ASSERT((CipherConcept)); - typename Cipher::EncryptionKey encKey = createRandomKey(); + typename Cipher::EncryptionKey encKey = createKeyFixture(); - static typename Cipher::EncryptionKey createRandomKey(int seed = 0) { + static typename Cipher::EncryptionKey createKeyFixture(int seed = 0) { Data data = DataFixture::generate(Cipher::EncryptionKey::BINARY_LENGTH, seed); return Cipher::EncryptionKey::FromBinary(data.data()); } @@ -40,6 +41,11 @@ public: EXPECT_EQ(Cipher::ciphertextSize(plaintext.size()), ciphertext.size()); } + void ExpectDoesntDecrypt(const Data &ciphertext) { + auto decrypted = Cipher::decrypt((byte*)ciphertext.data(), ciphertext.size(), this->encKey); + EXPECT_FALSE(decrypted); + } + Data Encrypt(const Data &plaintext) { return Cipher::encrypt((byte*)plaintext.data(), plaintext.size(), this->encKey); } @@ -103,23 +109,38 @@ TYPED_TEST_P(CipherTest, EncryptedSize) { } } +TYPED_TEST_P(CipherTest, TryDecryptDataThatIsTooSmall) { + Data tooSmallCiphertext(TypeParam::ciphertextSize(0) - 1); + this->ExpectDoesntDecrypt(tooSmallCiphertext); +} + +TYPED_TEST_P(CipherTest, TryDecryptDataThatIsMuchTooSmall_0) { + static_assert(TypeParam::ciphertextSize(0) > 0, "If this fails, the test case doesn't make sense."); + Data tooSmallCiphertext(0); + this->ExpectDoesntDecrypt(tooSmallCiphertext); +} + +TYPED_TEST_P(CipherTest, TryDecryptDataThatIsMuchTooSmall_1) { + static_assert(TypeParam::ciphertextSize(0) > 1, "If this fails, the test case doesn't make sense."); + Data tooSmallCiphertext(1); + this->ExpectDoesntDecrypt(tooSmallCiphertext); +} + REGISTER_TYPED_TEST_CASE_P(CipherTest, Size, EncryptThenDecrypt_Zeroes, EncryptThenDecrypt_Data, EncryptIsIndeterministic_Zeroes, EncryptIsIndeterministic_Data, - EncryptedSize + EncryptedSize, + TryDecryptDataThatIsTooSmall, + TryDecryptDataThatIsMuchTooSmall_0, + TryDecryptDataThatIsMuchTooSmall_1 ); template class AuthenticatedCipherTest: public CipherTest { public: - void ExpectDoesntDecrypt(const Data &ciphertext) { - auto decrypted = Cipher::decrypt((byte*)ciphertext.data(), ciphertext.size(), this->encKey); - EXPECT_FALSE(decrypted); - } - Data zeroes1 = CipherTest::CreateZeroes(1); Data plaintext1 = CipherTest::CreateData(1); Data zeroes2 = CipherTest::CreateZeroes(100 * 1024); @@ -184,23 +205,6 @@ TYPED_TEST_P(AuthenticatedCipherTest, TryDecryptRandomData) { this->ExpectDoesntDecrypt(this->plaintext2); } -TYPED_TEST_P(AuthenticatedCipherTest, TryDecryptDataThatIsTooSmall) { - Data tooSmallCiphertext(TypeParam::ciphertextSize(0) - 1); - this->ExpectDoesntDecrypt(tooSmallCiphertext); -} - -TYPED_TEST_P(AuthenticatedCipherTest, TryDecryptDataThatIsMuchTooSmall_0) { - static_assert(TypeParam::ciphertextSize(0) > 0, "If this fails, the test case doesn't make sense."); - Data tooSmallCiphertext(0); - this->ExpectDoesntDecrypt(tooSmallCiphertext); -} - -TYPED_TEST_P(AuthenticatedCipherTest, TryDecryptDataThatIsMuchTooSmall_1) { - static_assert(TypeParam::ciphertextSize(0) > 1, "If this fails, the test case doesn't make sense."); - Data tooSmallCiphertext(1); - this->ExpectDoesntDecrypt(tooSmallCiphertext); -} - REGISTER_TYPED_TEST_CASE_P(AuthenticatedCipherTest, ModifyFirstByte_Zeroes_Size1, ModifyFirstByte_Zeroes, @@ -211,13 +215,12 @@ REGISTER_TYPED_TEST_CASE_P(AuthenticatedCipherTest, ModifyMiddleByte_Zeroes, ModifyMiddleByte_Data, TryDecryptZeroesData, - TryDecryptRandomData, - TryDecryptDataThatIsTooSmall, - TryDecryptDataThatIsMuchTooSmall_0, - TryDecryptDataThatIsMuchTooSmall_1 + TryDecryptRandomData ); -INSTANTIATE_TYPED_TEST_CASE_P(AES256_CFB, CipherTest, AES256_CFB); +INSTANTIATE_TYPED_TEST_CASE_P(Fake, CipherTest, FakeAuthenticatedCipher); +INSTANTIATE_TYPED_TEST_CASE_P(Fake, AuthenticatedCipherTest, FakeAuthenticatedCipher); +INSTANTIATE_TYPED_TEST_CASE_P(AES256_CFB, CipherTest, AES256_CFB); //CFB mode is not authenticated INSTANTIATE_TYPED_TEST_CASE_P(AES256_GCM, CipherTest, AES256_GCM); INSTANTIATE_TYPED_TEST_CASE_P(AES256_GCM, AuthenticatedCipherTest, AES256_GCM); diff --git a/test/implementations/encrypted/EncryptedBlockStoreTest.cpp b/test/implementations/encrypted/EncryptedBlockStoreTest.cpp deleted file mode 100644 index 13c151a1..00000000 --- a/test/implementations/encrypted/EncryptedBlockStoreTest.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "../../../implementations/encrypted/ciphers/AES256_GCM.h" -#include "../../../implementations/encrypted/EncryptedBlockStore.h" -#include "../../../implementations/testfake/FakeBlockStore.h" -#include "../../testutils/BlockStoreTest.h" -#include "google/gtest/gtest.h" - - -using blockstore::BlockStore; -using blockstore::encrypted::EncryptedBlockStore; -using blockstore::testfake::FakeBlockStore; -using blockstore::encrypted::AES256_GCM; - -using std::unique_ptr; -using std::make_unique; - -class EncryptedBlockStoreTestFixture: public BlockStoreTestFixture { -public: - unique_ptr createBlockStore() override { - return make_unique>(make_unique(), AES256_GCM::EncryptionKey::FromString("1491BB4932A389EE14BC7090A272EE5517627CFA147A971A8E6E747E0C772972")); - } -}; - -INSTANTIATE_TYPED_TEST_CASE_P(Encrypted, BlockStoreTest, EncryptedBlockStoreTestFixture); - -//TODO Add specific tests, for example -// - loading it with a different encKey doesn't work -// - loading it with a different blockstore::Key will fail (because it stores its key in a header) -// - when using an authenticated cipher, loading a modified block will fail diff --git a/test/implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp b/test/implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp new file mode 100644 index 00000000..5507151d --- /dev/null +++ b/test/implementations/encrypted/EncryptedBlockStoreTest_Generic.cpp @@ -0,0 +1,40 @@ +#include "../../../implementations/encrypted/ciphers/AES256_GCM.h" +#include "../../../implementations/encrypted/ciphers/AES256_CFB.h" +#include "../../../implementations/encrypted/ciphers/Cipher.h" +#include "../../../implementations/encrypted/EncryptedBlockStore.h" +#include "../../../implementations/testfake/FakeBlockStore.h" +#include "../../testutils/BlockStoreTest.h" +#include "testutils/FakeAuthenticatedCipher.h" +#include "google/gtest/gtest.h" + +using ::testing::Test; + +using blockstore::BlockStore; +using blockstore::encrypted::EncryptedBlockStore; +using blockstore::testfake::FakeBlockStore; +using blockstore::encrypted::AES256_GCM; +using blockstore::encrypted::AES256_CFB; + +using std::unique_ptr; +using std::make_unique; + +using cpputils::Data; +using cpputils::DataFixture; + +template +class EncryptedBlockStoreTestFixture: public BlockStoreTestFixture { +public: + unique_ptr createBlockStore() override { + return make_unique>(make_unique(), createKeyFixture()); + } + +private: + static typename Cipher::EncryptionKey createKeyFixture(int seed = 0) { + Data data = DataFixture::generate(Cipher::EncryptionKey::BINARY_LENGTH, seed); + return Cipher::EncryptionKey::FromBinary(data.data()); + } +}; + +INSTANTIATE_TYPED_TEST_CASE_P(Encrypted_FakeCipher, BlockStoreTest, EncryptedBlockStoreTestFixture); +INSTANTIATE_TYPED_TEST_CASE_P(Encrypted_AES256_GCM, BlockStoreTest, EncryptedBlockStoreTestFixture); +INSTANTIATE_TYPED_TEST_CASE_P(Encrypted_AES256_CFB, BlockStoreTest, EncryptedBlockStoreTestFixture); diff --git a/test/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp b/test/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp new file mode 100644 index 00000000..072a028a --- /dev/null +++ b/test/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp @@ -0,0 +1,35 @@ +#include +#include "testutils/FakeAuthenticatedCipher.h" +#include "../../../implementations/encrypted/EncryptedBlockStore.h" +#include "../../../implementations/testfake/FakeBlockStore.h" + +using ::testing::Test; + +using std::unique_ptr; +using std::make_unique; + +using blockstore::testfake::FakeBlockStore; + +using namespace blockstore::encrypted; + +class EncryptedBlockStoreTest: public Test { +public: + EncryptedBlockStoreTest(): blockStore(make_unique>(make_unique(), FakeAuthenticatedCipher::Key1())) {} + unique_ptr> blockStore; +}; + +TEST_F(EncryptedBlockStoreTest, LoadingWithSameKeyWorks) { + //TODO implement +} + +TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentKeyDoesntWork) { + //TODO Implement +} + +TEST_F(EncryptedBlockStoreTest, LoadingModifiedBlockFails) { + //TODO Implement +} + +TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentBlockIdFails) { + //TODO loading it with a different blockstore::Key will fail (because it stores its key in a header) +} diff --git a/test/implementations/encrypted/testutils/FakeAuthenticatedCipher.cpp b/test/implementations/encrypted/testutils/FakeAuthenticatedCipher.cpp new file mode 100644 index 00000000..719ae3f0 --- /dev/null +++ b/test/implementations/encrypted/testutils/FakeAuthenticatedCipher.cpp @@ -0,0 +1,3 @@ +#include "FakeAuthenticatedCipher.h" + +constexpr unsigned int FakeKey::BINARY_LENGTH; diff --git a/test/implementations/encrypted/testutils/FakeAuthenticatedCipher.h b/test/implementations/encrypted/testutils/FakeAuthenticatedCipher.h new file mode 100644 index 00000000..652c983b --- /dev/null +++ b/test/implementations/encrypted/testutils/FakeAuthenticatedCipher.h @@ -0,0 +1,102 @@ +#pragma once +#ifndef MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_ENCRYPTED_TESTUTILS_FAKEAUTHENTICATEDCIPHER_H_ +#define MESSMER_BLOCKSTORE_TEST_IMPLEMENTATIONS_ENCRYPTED_TESTUTILS_FAKEAUTHENTICATEDCIPHER_H_ + +#include "../../../../implementations/encrypted/ciphers/Cipher.h" +#include + +struct FakeKey { + FakeKey(uint8_t value_):value(value_) {} + static FakeKey CreateRandom() { + return FakeKey(rand()); + } + static FakeKey FromBinary(const void *data) { + return FakeKey(*(uint8_t*)data); + } + static constexpr unsigned int BINARY_LENGTH = 1; + + uint8_t value; +}; + +// This is a fake cipher that uses an indeterministic caesar chiffre and a 4-byte parity for a simple authentication mechanism +class FakeAuthenticatedCipher { +public: + BOOST_CONCEPT_ASSERT((blockstore::encrypted::CipherConcept)); + + using EncryptionKey = FakeKey; + + static EncryptionKey Key1() { + return FakeKey(5); + } + static EncryptionKey Key2() { + return FakeKey(63); + } + + static constexpr unsigned int ciphertextSize(unsigned int plaintextBlockSize) { + return plaintextBlockSize + 5; + } + + static constexpr unsigned int plaintextSize(unsigned int ciphertextBlockSize) { + return ciphertextBlockSize - 5; + } + + static cpputils::Data encrypt(const byte *plaintext, unsigned int plaintextSize, const EncryptionKey &encKey) { + cpputils::Data result(ciphertextSize(plaintextSize)); + + //Add a random IV + uint8_t iv = rand(); + std::memcpy(result.data(), &iv, 1); + + //Use caesar chiffre on plaintext + _caesar((byte*)result.data() + 1, plaintext, plaintextSize, encKey.value + iv); + + //Add parity information + int32_t parity = _parity((byte*)result.data(), plaintextSize + 1); + std::memcpy((byte*)result.data() + plaintextSize + 1, &parity, 4); + + return result; + } + + static boost::optional decrypt(const byte *ciphertext, unsigned int ciphertextSize, const EncryptionKey &encKey) { + //We need at least 5 bytes (iv + parity) + if (ciphertextSize < 5) { + return boost::none; + } + + //Check parity + int32_t expectedParity = _parity(ciphertext, plaintextSize(ciphertextSize) + 1); + int32_t actualParity = *(int32_t*)(ciphertext + plaintextSize(ciphertextSize) + 1); + if (expectedParity != actualParity) { + return boost::none; + } + + //Decrypt caesar chiffre from ciphertext + int32_t iv = *(int32_t*)ciphertext; + cpputils::Data result(plaintextSize(ciphertextSize)); + _caesar((byte*)result.data(), ciphertext + 1, plaintextSize(ciphertextSize), -(encKey.value+iv)); + return std::move(result); + } + +private: + static int32_t _parity(const byte *data, unsigned int size) { + int32_t parity = 34343435; // some init value + int32_t *intData = (int32_t*)data; + unsigned int intSize = size / sizeof(int32_t); + for (unsigned int i = 0; i < intSize; ++i) { + parity += intData[i]; + } + unsigned int remainingBytes = size - 4 * intSize; + for (unsigned int i = 0; i < remainingBytes; ++i) { + parity += (data[4*intSize + i] << (24 - 8*i)); + } + return parity; + } + + static void _caesar(byte *dst, const byte *src, unsigned int size, uint8_t key) { + for (int i = 0; i < size; ++i) { + dst[i] = src[i] + key; + } + } +}; + +#endif From fdcd56401a4a2c6bc9330b7abb8ff7bedf904c10 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Wed, 6 May 2015 00:37:57 +0200 Subject: [PATCH 39/41] Added specific test cases for EncryptedBlockStore --- .../encrypted/EncryptedBlockStore.h | 8 ++ .../EncryptedBlockStoreTest_Specific.cpp | 96 +++++++++++++++++-- 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/implementations/encrypted/EncryptedBlockStore.h b/implementations/encrypted/EncryptedBlockStore.h index 09c35723..6845d3e4 100644 --- a/implementations/encrypted/EncryptedBlockStore.h +++ b/implementations/encrypted/EncryptedBlockStore.h @@ -22,6 +22,9 @@ public: void remove(std::unique_ptr block) override; uint64_t numBlocks() const override; + //This function should only be used by test cases + void __setKey(const typename Cipher::EncryptionKey &encKey); + private: std::unique_ptr _baseBlockStore; typename Cipher::EncryptionKey _encKey; @@ -66,6 +69,11 @@ uint64_t EncryptedBlockStore::numBlocks() const { return _baseBlockStore->numBlocks(); } +template +void EncryptedBlockStore::__setKey(const typename Cipher::EncryptionKey &encKey) { + _encKey = encKey; +} + } } diff --git a/test/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp b/test/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp index 072a028a..0a6ca336 100644 --- a/test/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp +++ b/test/implementations/encrypted/EncryptedBlockStoreTest_Specific.cpp @@ -2,34 +2,112 @@ #include "testutils/FakeAuthenticatedCipher.h" #include "../../../implementations/encrypted/EncryptedBlockStore.h" #include "../../../implementations/testfake/FakeBlockStore.h" +#include "../../../utils/BlockStoreUtils.h" +#include using ::testing::Test; using std::unique_ptr; using std::make_unique; +using cpputils::DataFixture; +using cpputils::Data; + using blockstore::testfake::FakeBlockStore; using namespace blockstore::encrypted; class EncryptedBlockStoreTest: public Test { public: - EncryptedBlockStoreTest(): blockStore(make_unique>(make_unique(), FakeAuthenticatedCipher::Key1())) {} + static constexpr unsigned int BLOCKSIZE = 1024; + EncryptedBlockStoreTest(): + _baseBlockStore(make_unique()), + baseBlockStore(_baseBlockStore.get()), + blockStore(make_unique>(std::move(_baseBlockStore), FakeAuthenticatedCipher::Key1())), + data(DataFixture::generate(BLOCKSIZE)) { + } + unique_ptr _baseBlockStore; + FakeBlockStore *baseBlockStore; unique_ptr> blockStore; + Data data; + + blockstore::Key CreateBlockDirectlyWithFixtureAndReturnKey() { + return blockStore->create(data)->key(); + } + + blockstore::Key CreateBlockWriteFixtureToItAndReturnKey() { + auto block = blockStore->create(Data(data.size())); + block->write(data.data(), 0, data.size()); + return block->key(); + } + + void ModifyBaseBlock(const blockstore::Key &key) { + auto block = baseBlockStore->load(key); + uint8_t middle_byte = ((byte*)block->data())[10]; + uint8_t new_middle_byte = middle_byte + 1; + block->write(&new_middle_byte, 10, 1); + } + + blockstore::Key CopyBaseBlock(const blockstore::Key &key) { + auto source = baseBlockStore->load(key); + return blockstore::utils::copyToNewBlock(baseBlockStore, *source)->key(); + } }; -TEST_F(EncryptedBlockStoreTest, LoadingWithSameKeyWorks) { - //TODO implement +TEST_F(EncryptedBlockStoreTest, LoadingWithSameKeyWorks_WriteOnCreate) { + auto key = CreateBlockDirectlyWithFixtureAndReturnKey(); + auto loaded = blockStore->load(key); + EXPECT_NE(nullptr, loaded.get()); + EXPECT_EQ(data.size(), loaded->size()); + EXPECT_EQ(0, std::memcmp(data.data(), loaded->data(), data.size())); } -TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentKeyDoesntWork) { - //TODO Implement +TEST_F(EncryptedBlockStoreTest, LoadingWithSameKeyWorks_WriteSeparately) { + auto key = CreateBlockWriteFixtureToItAndReturnKey(); + auto loaded = blockStore->load(key); + EXPECT_NE(nullptr, loaded.get()); + EXPECT_EQ(data.size(), loaded->size()); + EXPECT_EQ(0, std::memcmp(data.data(), loaded->data(), data.size())); } -TEST_F(EncryptedBlockStoreTest, LoadingModifiedBlockFails) { - //TODO Implement +TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentKeyDoesntWork_WriteOnCreate) { + auto key = CreateBlockDirectlyWithFixtureAndReturnKey(); + blockStore->__setKey(FakeAuthenticatedCipher::Key2()); + auto loaded = blockStore->load(key); + EXPECT_EQ(nullptr, loaded.get()); } -TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentBlockIdFails) { - //TODO loading it with a different blockstore::Key will fail (because it stores its key in a header) +TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentKeyDoesntWork_WriteSeparately) { + auto key = CreateBlockWriteFixtureToItAndReturnKey(); + blockStore->__setKey(FakeAuthenticatedCipher::Key2()); + auto loaded = blockStore->load(key); + EXPECT_EQ(nullptr, loaded.get()); +} + +TEST_F(EncryptedBlockStoreTest, LoadingModifiedBlockFails_WriteOnCreate) { + auto key = CreateBlockDirectlyWithFixtureAndReturnKey(); + ModifyBaseBlock(key); + auto loaded = blockStore->load(key); + EXPECT_EQ(nullptr, loaded.get()); +} + +TEST_F(EncryptedBlockStoreTest, LoadingModifiedBlockFails_WriteSeparately) { + auto key = CreateBlockWriteFixtureToItAndReturnKey(); + ModifyBaseBlock(key); + auto loaded = blockStore->load(key); + EXPECT_EQ(nullptr, loaded.get()); +} + +TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentBlockIdFails_WriteOnCreate) { + auto key = CreateBlockDirectlyWithFixtureAndReturnKey(); + auto key2 = CopyBaseBlock(key); + auto loaded = blockStore->load(key2); + EXPECT_EQ(nullptr, loaded.get()); +} + +TEST_F(EncryptedBlockStoreTest, LoadingWithDifferentBlockIdFails_WriteSeparately) { + auto key = CreateBlockWriteFixtureToItAndReturnKey(); + auto key2 = CopyBaseBlock(key); + auto loaded = blockStore->load(key2); + EXPECT_EQ(nullptr, loaded.get()); } From 535bfc16347fd855eb5c7c3cf1cd7ad91435848d Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Wed, 6 May 2015 16:16:18 +0200 Subject: [PATCH 40/41] TODOs --- implementations/encrypted/EncryptedBlock.h | 2 ++ implementations/encrypted/EncryptedBlockStore.h | 3 +++ 2 files changed, 5 insertions(+) diff --git a/implementations/encrypted/EncryptedBlock.h b/implementations/encrypted/EncryptedBlock.h index 3770786e..f7b96f5b 100644 --- a/implementations/encrypted/EncryptedBlock.h +++ b/implementations/encrypted/EncryptedBlock.h @@ -16,6 +16,8 @@ namespace blockstore { namespace encrypted { template class EncryptedBlockStore; +//TODO Test EncryptedBlock + template class EncryptedBlock: public Block { public: diff --git a/implementations/encrypted/EncryptedBlockStore.h b/implementations/encrypted/EncryptedBlockStore.h index 6845d3e4..ad7acd31 100644 --- a/implementations/encrypted/EncryptedBlockStore.h +++ b/implementations/encrypted/EncryptedBlockStore.h @@ -16,6 +16,7 @@ class EncryptedBlockStore: public BlockStore { public: EncryptedBlockStore(std::unique_ptr baseBlockStore, const typename Cipher::EncryptionKey &encKey); + //TODO Are createKey() tests included in generic BlockStoreTest? If not, add it! Key createKey() override; std::unique_ptr tryCreate(const Key &key, cpputils::Data data) override; std::unique_ptr load(const Key &key) override; @@ -46,6 +47,7 @@ Key EncryptedBlockStore::createKey() { template std::unique_ptr EncryptedBlockStore::tryCreate(const Key &key, cpputils::Data data) { + //TODO Test that this returns nullptr when base blockstore returns nullptr (for all pass-through-blockstores) return EncryptedBlock::TryCreateNew(_baseBlockStore.get(), key, std::move(data), _encKey); } @@ -53,6 +55,7 @@ template std::unique_ptr EncryptedBlockStore::load(const Key &key) { auto block = _baseBlockStore->load(key); if (block.get() == nullptr) { + //TODO Test this path (for all pass-through-blockstores) return nullptr; } return EncryptedBlock::TryDecrypt(std::move(block), _encKey); From e8a48afd3b439aa6ccf343e12bb6555f5d75aee7 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 8 May 2015 02:10:56 +0200 Subject: [PATCH 41/41] Enable extended compiler warnings and resolve occurring ones --- CMakeLists.txt | 1 + .../cache/CacheTest_MoveConstructor.cpp | 2 ++ .../caching/cache/CacheTest_PushAndPop.cpp | 26 +++++++++---------- .../cache/QueueMapTest_MoveConstructor.cpp | 4 +++ .../testutils/FakeAuthenticatedCipher.h | 2 +- .../helpers/BlockStoreWithRandomKeysTest.cpp | 2 +- 6 files changed, 22 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b345e659..b0034179 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ ADD_BOOST(filesystem system thread) TARGET_LINK_LIBRARIES(${BII_BLOCK_TARGET} INTERFACE rt) ACTIVATE_CPP14() +ENABLE_STYLE_WARNINGS() # You can safely delete lines from here... diff --git a/test/implementations/caching/cache/CacheTest_MoveConstructor.cpp b/test/implementations/caching/cache/CacheTest_MoveConstructor.cpp index 925ab99f..5efba4a8 100644 --- a/test/implementations/caching/cache/CacheTest_MoveConstructor.cpp +++ b/test/implementations/caching/cache/CacheTest_MoveConstructor.cpp @@ -23,6 +23,7 @@ public: TEST_F(CacheTest_MoveConstructor, MoveIntoCache) { cache->push(MinimalKeyType::create(0), CopyableMovableValueType(2)); CopyableMovableValueType val = cache->pop(MinimalKeyType::create(0)).value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled); } @@ -30,5 +31,6 @@ TEST_F(CacheTest_MoveConstructor, CopyIntoCache) { CopyableMovableValueType value(2); cache->push(MinimalKeyType::create(0), value); CopyableMovableValueType val = cache->pop(MinimalKeyType::create(0)).value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled); } diff --git a/test/implementations/caching/cache/CacheTest_PushAndPop.cpp b/test/implementations/caching/cache/CacheTest_PushAndPop.cpp index 6203fe1a..c64cc7b9 100644 --- a/test/implementations/caching/cache/CacheTest_PushAndPop.cpp +++ b/test/implementations/caching/cache/CacheTest_PushAndPop.cpp @@ -21,7 +21,7 @@ TEST_F(CacheTest_PushAndPop, PopNonExistingEntry_NonEmptyCache) { TEST_F(CacheTest_PushAndPop, PopNonExistingEntry_FullCache) { //Add a lot of even numbered keys - for (int i = 0; i < Cache::MAX_ENTRIES; ++i) { + for (unsigned int i = 0; i < Cache::MAX_ENTRIES; ++i) { push(2*i, 2*i); } //Request an odd numbered key @@ -43,34 +43,34 @@ TEST_F(CacheTest_PushAndPop, MultipleEntries) { } TEST_F(CacheTest_PushAndPop, FullCache) { - for(int i = 0; i < Cache::MAX_ENTRIES; ++i) { + for(unsigned int i = 0; i < Cache::MAX_ENTRIES; ++i) { push(i, 2*i); } - for(int i = 0; i < Cache::MAX_ENTRIES; ++i) { + for(unsigned int i = 0; i < Cache::MAX_ENTRIES; ++i) { EXPECT_EQ(2*i, pop(i).value()); } } TEST_F(CacheTest_PushAndPop, FullCache_PushNonOrdered_PopOrdered) { - for(int i = 1; i < Cache::MAX_ENTRIES; i += 2) { + for(unsigned int i = 1; i < Cache::MAX_ENTRIES; i += 2) { push(i, 2*i); } - for(int i = 0; i < Cache::MAX_ENTRIES; i += 2) { + for(unsigned int i = 0; i < Cache::MAX_ENTRIES; i += 2) { push(i, 2*i); } - for(int i = 0; i < Cache::MAX_ENTRIES; ++i) { + for(unsigned int i = 0; i < Cache::MAX_ENTRIES; ++i) { EXPECT_EQ(2*i, pop(i).value()); } } TEST_F(CacheTest_PushAndPop, FullCache_PushOrdered_PopNonOrdered) { - for(int i = 0; i < Cache::MAX_ENTRIES; ++i) { + for(unsigned int i = 0; i < Cache::MAX_ENTRIES; ++i) { push(i, 2*i); } - for(int i = 1; i < Cache::MAX_ENTRIES; i += 2) { + for(unsigned int i = 1; i < Cache::MAX_ENTRIES; i += 2) { EXPECT_EQ(2*i, pop(i).value()); } - for(int i = 0; i < Cache::MAX_ENTRIES; i += 2) { + for(unsigned int i = 0; i < Cache::MAX_ENTRIES; i += 2) { EXPECT_EQ(2*i, pop(i).value()); } } @@ -95,26 +95,26 @@ TEST_F(CacheTest_PushAndPop, FullCache_PushNonOrdered_PopNonOrdered) { for(int i = roundDownToEven(Cache::MAX_ENTRIES - 1); i >= 0; i -= 2) { push(i, 2*i); } - for(int i = 1; i < Cache::MAX_ENTRIES; i += 2) { + for(unsigned int i = 1; i < Cache::MAX_ENTRIES; i += 2) { push(i, 2*i); } for(int i = roundDownToOdd(Cache::MAX_ENTRIES-1); i >= 0; i -= 2) { EXPECT_EQ(2*i, pop(i).value()); } - for(int i = 0; i < Cache::MAX_ENTRIES; i += 2) { + for(unsigned int i = 0; i < Cache::MAX_ENTRIES; i += 2) { EXPECT_EQ(2*i, pop(i).value()); } } TEST_F(CacheTest_PushAndPop, MoreThanFullCache) { - for(int i = 0; i < Cache::MAX_ENTRIES + 2; ++i) { + for(unsigned int i = 0; i < Cache::MAX_ENTRIES + 2; ++i) { push(i, 2*i); } //Check that the oldest two elements got deleted automatically EXPECT_EQ(boost::none, pop(0)); EXPECT_EQ(boost::none, pop(1)); //Check the other elements are still there - for(int i = 2; i < Cache::MAX_ENTRIES + 2; ++i) { + for(unsigned int i = 2; i < Cache::MAX_ENTRIES + 2; ++i) { EXPECT_EQ(2*i, pop(i).value()); } } diff --git a/test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp b/test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp index c0bebd71..4a1f15db 100644 --- a/test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp +++ b/test/implementations/caching/cache/QueueMapTest_MoveConstructor.cpp @@ -23,12 +23,14 @@ public: TEST_F(QueueMapTest_MoveConstructor, PushingAndPopping_MoveIntoMap) { map->push(MinimalKeyType::create(0), CopyableMovableValueType(2)); CopyableMovableValueType val = map->pop().value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled); } TEST_F(QueueMapTest_MoveConstructor, PushingAndPoppingPerKey_MoveIntoMap) { map->push(MinimalKeyType::create(0), CopyableMovableValueType(2)); CopyableMovableValueType val = map->pop(MinimalKeyType::create(0)).value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away EXPECT_EQ(0, CopyableMovableValueType::numCopyConstructorCalled); } @@ -36,6 +38,7 @@ TEST_F(QueueMapTest_MoveConstructor, PushingAndPopping_CopyIntoMap) { CopyableMovableValueType value(2); map->push(MinimalKeyType::create(0), value); CopyableMovableValueType val = map->pop().value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled); } @@ -43,5 +46,6 @@ TEST_F(QueueMapTest_MoveConstructor, PushingAndPoppingPerKey_CopyIntoMap) { CopyableMovableValueType value(2); map->push(MinimalKeyType::create(0), value); CopyableMovableValueType val = map->pop(MinimalKeyType::create(0)).value(); + val.value(); //Access it to avoid the compiler optimizing the assignment away EXPECT_EQ(1, CopyableMovableValueType::numCopyConstructorCalled); } diff --git a/test/implementations/encrypted/testutils/FakeAuthenticatedCipher.h b/test/implementations/encrypted/testutils/FakeAuthenticatedCipher.h index 652c983b..d99ff86d 100644 --- a/test/implementations/encrypted/testutils/FakeAuthenticatedCipher.h +++ b/test/implementations/encrypted/testutils/FakeAuthenticatedCipher.h @@ -93,7 +93,7 @@ private: } static void _caesar(byte *dst, const byte *src, unsigned int size, uint8_t key) { - for (int i = 0; i < size; ++i) { + for (unsigned int i = 0; i < size; ++i) { dst[i] = src[i] + key; } } diff --git a/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp b/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp index d4c0513e..fdcfada0 100644 --- a/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp +++ b/test/interface/helpers/BlockStoreWithRandomKeysTest.cpp @@ -28,7 +28,7 @@ public: return unique_ptr(do_load(key)); } MOCK_METHOD1(do_load, Block*(const Key &)); - void remove(unique_ptr block) {} + void remove(unique_ptr block) {UNUSED(block);} MOCK_CONST_METHOD0(numBlocks, uint64_t()); };