hardcoresushi/libcryfs
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Implement BlockStore::overwrite(). This is the last step in ensuring that the write() call doen't have to load leaves if they're only overwritten anyhow.

Browse Source
This commit is contained in:
Sebastian Messmer 2016-07-16 11:42:06 +02:00
parent eb792daefc
commit eab7cb1df4
38 changed files with 295 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/blobstore/implementations/onblocks/BlobOnBlocks.cpp
View File

@ -2,6 +2,7 @@
#include "BlobOnBlocks.h"
#include "datanodestore/DataLeafNode.h"
#include "datanodestore/DataNodeStore.h"
#include "utils/Math.h"
#include <cmath>
#include <cpp-utils/assert/assert.h>
@ -127,7 +128,14 @@ void BlobOnBlocks::write(const void *source, uint64_t offset, uint64_t count) {
auto onExistingLeaf = [source, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Reading from source out of bounds");
//TODO Simplify formula, make it easier to understand
leaf.node()->write((uint8_t*)source + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
if (leafDataOffset == 0 && leafDataSize == leaf.nodeStore()->layout().maxBytesPerLeaf()) {
Data leafData(leafDataSize);
std::memcpy(leafData.data(), (uint8_t*)source + indexOfFirstLeafByte - offset, leafDataSize);
leaf.nodeStore()->overwriteLeaf(leaf.key(), std::move(leafData));
} else {
leaf.node()->write((uint8_t *) source + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset,
leafDataSize);
}
};
auto onCreateLeaf = [source, offset, count] (uint64_t beginByte, uint32_t numBytes) -> Data {
ASSERT(beginByte >= offset && beginByte-offset <= count && beginByte-offset+numBytes <= count, "Reading from source out of bounds");

6
src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.cpp
View File

@ -31,6 +31,12 @@ unique_ref<DataLeafNode> DataLeafNode::CreateNewNode(BlockStore *blockStore, con
return make_unique_ref<DataLeafNode>(DataNodeView::create(blockStore, layout, DataNode::FORMAT_VERSION_HEADER, 0, size, std::move(data)));
}
unique_ref<DataLeafNode> DataLeafNode::OverwriteNode(BlockStore *blockStore, const DataNodeLayout &layout, const Key &key, Data data) {
ASSERT(data.size() == layout.maxBytesPerLeaf(), "Data passed in is too large for one leaf.");
uint32_t size = data.size();
return make_unique_ref<DataLeafNode>(DataNodeView::overwrite(blockStore, layout, DataNode::FORMAT_VERSION_HEADER, 0, size, key, std::move(data)));
}
void DataLeafNode::read(void *target, uint64_t offset, uint64_t size) const {
ASSERT(offset <= node().Size() && offset + size <= node().Size(), "Read out of valid area"); // Also check offset, because the addition could lead to overflows
std::memcpy(target, (uint8_t*)node().data() + offset, size);

2
src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.h
View File

@ -11,8 +11,8 @@ class DataInnerNode;
class DataLeafNode final: public DataNode {
public:
//static cpputils::unique_ref<DataLeafNode> InitializeNewNode(cpputils::unique_ref<blockstore::Block> block);
static cpputils::unique_ref<DataLeafNode> CreateNewNode(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, cpputils::Data data);
static cpputils::unique_ref<DataLeafNode> OverwriteNode(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, const blockstore::Key &key, cpputils::Data data);
DataLeafNode(DataNodeView block);
~DataLeafNode();

4
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp
View File

@ -50,6 +50,10 @@ unique_ref<DataLeafNode> DataNodeStore::createNewLeafNode(Data data) {
return DataLeafNode::CreateNewNode(_blockstore.get(), _layout, std::move(data));
}
unique_ref<DataLeafNode> DataNodeStore::overwriteLeaf(const Key &key, Data data) {
return DataLeafNode::OverwriteNode(_blockstore.get(), _layout, key, std::move(data));
}
optional<unique_ref<DataNode>> DataNodeStore::load(const Key &key) {
auto block = _blockstore->load(key);
if (block == none) {

2
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h
View File

@ -38,6 +38,8 @@ public:
cpputils::unique_ref<DataNode> overwriteNodeWith(cpputils::unique_ref<DataNode> target, const DataNode &source);
cpputils::unique_ref<DataLeafNode> overwriteLeaf(const blockstore::Key &key, cpputils::Data data);
void remove(cpputils::unique_ref<DataNode> node);
void remove(const blockstore::Key &key);
void removeSubtree(uint8_t depth, const blockstore::Key &key);

7
src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h
View File

@ -81,6 +81,13 @@ public:
return DataNodeView(std::move(block));
}
static DataNodeView overwrite(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, uint16_t formatVersion, uint8_t depth, uint32_t size, const blockstore::Key &key, cpputils::Data data) {
ASSERT(data.size() <= layout.datasizeBytes(), "Data is too large for node");
cpputils::Data serialized = _serialize(layout, formatVersion, depth, size, std::move(data));
auto block = blockStore->overwrite(key, std::move(serialized));
return DataNodeView(std::move(block));
}
DataNodeView(DataNodeView &&rhs) = default;
uint16_t FormatVersion() const {

4
src/blobstore/implementations/onblocks/datatreestore/LeafHandle.h
View File

@ -26,6 +26,10 @@ namespace blobstore {
datanodestore::DataLeafNode *node();
datanodestore::DataNodeStore *nodeStore() {
return _nodeStore;
}
private:
datanodestore::DataNodeStore *_nodeStore;
blockstore::Key _key;

14
src/blockstore/implementations/caching/CachingBlockStore.cpp
View File

@ -49,6 +49,20 @@ optional<unique_ref<Block>> CachingBlockStore::load(const Key &key) {
}
}
unique_ref<Block> CachingBlockStore::overwrite(const Key &key, cpputils::Data data) {
optional<unique_ref<Block>> optBlock = _cache.pop(key);
if (optBlock != none) {
if ((*optBlock)->size() != data.size()) {
(*optBlock)->resize(data.size());
}
(*optBlock)->write(data.data(), 0, data.size());
return make_unique_ref<CachedBlock>(std::move(*optBlock), this);
} else {
auto block = _baseBlockStore->overwrite(key, std::move(data));
return make_unique_ref<CachedBlock>(std::move(block), this);
}
}
void CachingBlockStore::remove(cpputils::unique_ref<Block> block) {
auto cached_block = dynamic_pointer_move<CachedBlock>(block);
ASSERT(cached_block != none, "Passed block is not a CachedBlock");

1
src/blockstore/implementations/caching/CachingBlockStore.h
View File

@ -18,6 +18,7 @@ public:
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
cpputils::unique_ref<Block> overwrite(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;

9
src/blockstore/implementations/compressing/CompressedBlock.h
View File

@ -17,6 +17,7 @@ template<class Compressor>
class CompressedBlock final: public Block {
public:
static boost::optional<cpputils::unique_ref<CompressedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData);
static cpputils::unique_ref<CompressedBlock> Overwrite(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData);
static cpputils::unique_ref<CompressedBlock> Decompress(cpputils::unique_ref<Block> baseBlock);
CompressedBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data decompressedData);
@ -55,6 +56,14 @@ boost::optional<cpputils::unique_ref<CompressedBlock<Compressor>>> CompressedBlo
return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(*baseBlock), std::move(decompressedData));
}
template<class Compressor>
cpputils::unique_ref<CompressedBlock<Compressor>> CompressedBlock<Compressor>::Overwrite(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData) {
cpputils::Data compressed = Compressor::Compress(decompressedData);
auto baseBlock = baseBlockStore->overwrite(key, std::move(compressed));
return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(baseBlock), std::move(decompressedData));
}
template<class Compressor>
cpputils::unique_ref<CompressedBlock<Compressor>> CompressedBlock<Compressor>::Decompress(cpputils::unique_ref<Block> baseBlock) {
cpputils::Data decompressed = Compressor::Decompress((byte*)baseBlock->data(), baseBlock->size());

6
src/blockstore/implementations/compressing/CompressingBlockStore.h
View File

@ -17,6 +17,7 @@ public:
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
cpputils::unique_ref<Block> overwrite(const blockstore::Key &key, cpputils::Data data) override;
void remove(const Key &key) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
@ -52,6 +53,11 @@ boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::
return cpputils::unique_ref<Block>(std::move(*result));
}
template<class Compressor>
cpputils::unique_ref<Block> CompressingBlockStore<Compressor>::overwrite(const blockstore::Key &key, cpputils::Data data) {
return CompressedBlock<Compressor>::Overwrite(_baseBlockStore.get(), key, std::move(data));
}
template<class Compressor>
boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::load(const Key &key) {
auto loaded = _baseBlockStore->load(key);

12
src/blockstore/implementations/encrypted/EncryptedBlock.h
View File

@ -29,6 +29,7 @@ class EncryptedBlock final: public Block {
public:
BOOST_CONCEPT_ASSERT((cpputils::CipherConcept<Cipher>));
static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey);
static cpputils::unique_ref<EncryptedBlock> Overwrite(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey);
static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key);
static uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize);
@ -91,6 +92,17 @@ boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cip
return cpputils::make_unique_ref<EncryptedBlock>(std::move(*baseBlock), encKey, std::move(plaintextWithHeader));
}
template<class Cipher>
cpputils::unique_ref<EncryptedBlock<Cipher>> EncryptedBlock<Cipher>::Overwrite(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey) {
//TODO Is it possible to avoid copying the whole plaintext data into plaintextWithHeader? Maybe an encrypt() object that has an .addData() function and concatenates all data for encryption? Maybe Crypto++ offers this functionality already.
cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, std::move(data));
cpputils::Data encrypted = Cipher::encrypt((byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), encKey);
//TODO Avoid copying the whole encrypted block into a encryptedWithFormatHeader by creating a Data object with full size and then giving it as an encryption target to Cipher::encrypt()
cpputils::Data encryptedWithFormatHeader = _prependFormatHeader(std::move(encrypted));
auto baseBlock = baseBlockStore->overwrite(key, std::move(encryptedWithFormatHeader));
return cpputils::make_unique_ref<EncryptedBlock>(std::move(baseBlock), encKey, std::move(plaintextWithHeader));
}
template<class Cipher>
cpputils::Data EncryptedBlock<Cipher>::_prependFormatHeader(const cpputils::Data &data) {
cpputils::Data dataWithHeader(sizeof(FORMAT_VERSION_HEADER) + data.size());

7
src/blockstore/implementations/encrypted/EncryptedBlockStore.h
View File

@ -20,6 +20,7 @@ public:
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
cpputils::unique_ref<Block> overwrite(const blockstore::Key &key, cpputils::Data data) override;
void remove(const Key &key) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
@ -37,7 +38,6 @@ private:
};
template<class Cipher>
EncryptedBlockStore<Cipher>::EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
: _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
@ -69,6 +69,11 @@ boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::load(c
return boost::optional<cpputils::unique_ref<Block>>(EncryptedBlock<Cipher>::TryDecrypt(std::move(*block), _encKey));
}
template<class Cipher>
cpputils::unique_ref<Block> EncryptedBlockStore<Cipher>::overwrite(const blockstore::Key &key, cpputils::Data data) {
return EncryptedBlock<Cipher>::Overwrite(_baseBlockStore.get(), key, std::move(data), _encKey);
}
template<class Cipher>
void EncryptedBlockStore<Cipher>::remove(const Key &key) {
return _baseBlockStore->remove(key);

5
src/blockstore/implementations/inmemory/InMemoryBlock.cpp
View File

@ -26,6 +26,11 @@ InMemoryBlock::InMemoryBlock(const InMemoryBlock &rhs)
InMemoryBlock::~InMemoryBlock() {
}
InMemoryBlock &InMemoryBlock::operator=(const InMemoryBlock &rhs) {
_data = rhs._data;
return *this;
}
const void *InMemoryBlock::data() const {
return _data->data();
}

1
src/blockstore/implementations/inmemory/InMemoryBlock.h
View File

@ -14,6 +14,7 @@ public:
InMemoryBlock(const Key &key, cpputils::Data size);
InMemoryBlock(const InMemoryBlock &rhs);
~InMemoryBlock();
InMemoryBlock &operator=(const InMemoryBlock &rhs);
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;

13
src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp
View File

@ -42,6 +42,19 @@ optional<unique_ref<Block>> InMemoryBlockStore::load(const Key &key) {
}
}
unique_ref<Block> InMemoryBlockStore::overwrite(const Key &key, Data data) {
InMemoryBlock newBlock(key, std::move(data));
auto insert_result = _blocks.emplace(key, newBlock);
if (!insert_result.second) {
// If block already exists, overwrite it.
insert_result.first->second = newBlock;
}
//Return a pointer to the stored InMemoryBlock
return make_unique_ref<InMemoryBlock>(insert_result.first->second);
}
void InMemoryBlockStore::remove(const Key &key) {
int numRemoved = _blocks.erase(key);
ASSERT(1==numRemoved, "Didn't find block to remove");

1
src/blockstore/implementations/inmemory/InMemoryBlockStore.h
View File

@ -19,6 +19,7 @@ public:
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
cpputils::unique_ref<Block> overwrite(const blockstore::Key &key, cpputils::Data data) override;
void remove(const Key &key) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;

5
src/blockstore/implementations/mock/MockBlockStore.h
View File

@ -41,6 +41,11 @@ namespace blockstore {
return boost::optional<cpputils::unique_ref<Block>>(cpputils::make_unique_ref<MockBlock>(std::move(*base), this));
}
cpputils::unique_ref<Block> overwrite(const Key &key, cpputils::Data data) override {
_increaseNumWrittenBlocks(key);
return _baseBlockStore->overwrite(key, std::move(data));
}
void remove(const Key &key) override {
_increaseNumRemovedBlocks(key);
return _baseBlockStore->remove(key);

8
src/blockstore/implementations/ondisk/OnDiskBlock.cpp
View File

@ -86,6 +86,14 @@ optional<unique_ref<OnDiskBlock>> OnDiskBlock::CreateOnDisk(const bf::path &root
return std::move(block);
}
unique_ref<OnDiskBlock> OnDiskBlock::OverwriteOnDisk(const bf::path &rootdir, const Key &key, Data data) {
auto filepath = _getFilepath(rootdir, key);
bf::create_directory(filepath.parent_path());
auto block = make_unique_ref<OnDiskBlock>(key, filepath, std::move(data));
block->_storeToDisk();
return std::move(block);
}
void OnDiskBlock::RemoveFromDisk(const bf::path &rootdir, const Key &key) {
auto filepath = _getFilepath(rootdir, key);
ASSERT(bf::is_regular_file(filepath), "Block not found on disk");

1
src/blockstore/implementations/ondisk/OnDiskBlock.h
View File

@ -26,6 +26,7 @@ public:
static boost::optional<cpputils::unique_ref<OnDiskBlock>> LoadFromDisk(const boost::filesystem::path &rootdir, const Key &key);
static boost::optional<cpputils::unique_ref<OnDiskBlock>> CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data);
static cpputils::unique_ref<OnDiskBlock> OverwriteOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data);
static void RemoveFromDisk(const boost::filesystem::path &rootdir, const Key &key);
const void *data() const override;

4
src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp
View File

@ -65,6 +65,10 @@ optional<unique_ref<Block>> OnDiskBlockStore::tryCreate(const Key &key, Data dat
return unique_ref<Block>(std::move(*result));
}
unique_ref<Block> OnDiskBlockStore::overwrite(const Key &key, Data data) {
return OnDiskBlock::OverwriteOnDisk(_rootdir, key, std::move(data));
}
optional<unique_ref<Block>> OnDiskBlockStore::load(const Key &key) {
return optional<unique_ref<Block>>(OnDiskBlock::LoadFromDisk(_rootdir, key));
}

1
src/blockstore/implementations/ondisk/OnDiskBlockStore.h
View File

@ -22,6 +22,7 @@ public:
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const Key &)> callback) const override;
cpputils::unique_ref<Block> overwrite(const blockstore::Key &key, cpputils::Data data) override;
private:
const boost::filesystem::path _rootdir;

15
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.cpp
View File

@ -12,6 +12,7 @@ using cpputils::make_unique_ref;
using boost::none;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using cpputils::Data;
using boost::optional;
using boost::none;
@ -26,7 +27,7 @@ Key ParallelAccessBlockStore::createKey() {
return _baseBlockStore->createKey();
}
optional<unique_ref<Block>> ParallelAccessBlockStore::tryCreate(const Key &key, cpputils::Data data) {
optional<unique_ref<Block>> ParallelAccessBlockStore::tryCreate(const Key &key, Data data) {
ASSERT(!_parallelAccessStore.isOpened(key), ("Key "+key.ToString()+"already exists").c_str());
auto block = _baseBlockStore->tryCreate(key, std::move(data));
if (block == none) {
@ -44,6 +45,18 @@ optional<unique_ref<Block>> ParallelAccessBlockStore::load(const Key &key) {
return unique_ref<Block>(std::move(*block));
}
unique_ref<Block> ParallelAccessBlockStore::overwrite(const Key &key, Data data) {
auto onExists = [&data] (BlockRef *block) {
if (block->size() != data.size()) {
block->resize(data.size());
}
block->write(data.data(), 0, data.size());
};
auto onAdd = [this, key, &data] {
return _baseBlockStore->overwrite(key, data.copy()); // TODO Without copy?
};
return _parallelAccessStore.loadOrAdd(key, onExists, onAdd);
}
void ParallelAccessBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();

1
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h
View File

@ -18,6 +18,7 @@ public:
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
cpputils::unique_ref<Block> overwrite(const Key &key, cpputils::Data data) override;
void remove(const Key &key) override;
void remove(cpputils::unique_ref<Block> node) override;
uint64_t numBlocks() const override;

15
src/blockstore/implementations/testfake/FakeBlockStore.cpp
View File

@ -31,6 +31,21 @@ optional<unique_ref<Block>> FakeBlockStore::tryCreate(const Key &key, Data data)
return _load(key);
}
unique_ref<Block> FakeBlockStore::overwrite(const Key &key, Data data) {
std::unique_lock<std::mutex> lock(_mutex);
auto insert_result = _blocks.emplace(key, data.copy());
if (!insert_result.second) {
// If block already exists, overwrite it.
insert_result.first->second = std::move(data);
}
//Return a pointer to the stored FakeBlock
auto loaded = _load(key);
ASSERT(loaded != none, "Block was just created or written. Should exist.");
return std::move(*loaded);
}
optional<unique_ref<Block>> FakeBlockStore::load(const Key &key) {
std::unique_lock<std::mutex> lock(_mutex);
return _load(key);

1
src/blockstore/implementations/testfake/FakeBlockStore.h
View File

@ -32,6 +32,7 @@ public:
FakeBlockStore();
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
cpputils::unique_ref<Block> overwrite(const blockstore::Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(const Key &key) override;
uint64_t numBlocks() const override;

4
src/blockstore/implementations/versioncounting/KnownBlockVersions.cpp
View File

@ -72,10 +72,10 @@ bool KnownBlockVersions::checkAndUpdateVersion(uint32_t clientId, const Key &key
return true;
}
uint64_t KnownBlockVersions::incrementVersion(const Key &key, uint64_t lastVersion) {
uint64_t KnownBlockVersions::incrementVersion(const Key &key) {
unique_lock<mutex> lock(_mutex);
uint64_t &found = _knownVersions[{_myClientId, key}]; // If the entry doesn't exist, this creates it with value 0.
uint64_t newVersion = std::max(lastVersion + 1, found + 1);
uint64_t newVersion = found + 1;
if (newVersion == std::numeric_limits<uint64_t>::max()) {
// It's *very* unlikely we ever run out of version numbers in 64bit...but just to be sure...
throw std::runtime_error("Version overflow");

2
src/blockstore/implementations/versioncounting/KnownBlockVersions.h
View File

@ -24,7 +24,7 @@ namespace blockstore {
__attribute__((warn_unused_result))
bool checkAndUpdateVersion(uint32_t clientId, const Key &key, uint64_t version);
uint64_t incrementVersion(const Key &key, uint64_t lastVersion);
uint64_t incrementVersion(const Key &key);
void markBlockAsDeleted(const Key &key);

2
src/blockstore/implementations/versioncounting/VersionCountingBlock.cpp
View File

@ -10,7 +10,7 @@ namespace blockstore {
#ifndef CRYFS_NO_COMPATIBILITY
void VersionCountingBlock::migrateFromBlockstoreWithoutVersionNumbers(cpputils::unique_ref<Block> baseBlock, KnownBlockVersions *knownBlockVersions) {
uint64_t version = knownBlockVersions->incrementVersion(baseBlock->key(), VERSION_ZERO);
uint64_t version = knownBlockVersions->incrementVersion(baseBlock->key());
cpputils::Data data(baseBlock->size());
std::memcpy(data.data(), baseBlock->data(), data.size());

19
src/blockstore/implementations/versioncounting/VersionCountingBlock.h
View File

@ -28,6 +28,7 @@ namespace versioncounting {
class VersionCountingBlock final: public Block {
public:
static boost::optional<cpputils::unique_ref<VersionCountingBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, VersionCountingBlockStore *blockStore);
static cpputils::unique_ref<VersionCountingBlock> Overwrite(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, VersionCountingBlockStore *blockStore);
static cpputils::unique_ref<VersionCountingBlock> Load(cpputils::unique_ref<Block> baseBlock, VersionCountingBlockStore *blockStore);
static uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize);
@ -53,7 +54,6 @@ private:
VersionCountingBlockStore *_blockStore;
cpputils::unique_ref<Block> _baseBlock;
cpputils::Data _dataWithHeader;
uint64_t _version;
bool _dataChanged;
std::mutex _mutex;
@ -78,7 +78,7 @@ public:
inline boost::optional<cpputils::unique_ref<VersionCountingBlock>> VersionCountingBlock::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, VersionCountingBlockStore *blockStore) {
uint64_t version = blockStore->knownBlockVersions()->incrementVersion(key, VERSION_ZERO);
uint64_t version = blockStore->knownBlockVersions()->incrementVersion(key);
cpputils::Data dataWithHeader = _prependHeaderToData(blockStore->knownBlockVersions()->myClientId(), version, std::move(data));
auto baseBlock = baseBlockStore->tryCreate(key, dataWithHeader.copy()); // TODO Copy necessary?
@ -90,6 +90,14 @@ inline boost::optional<cpputils::unique_ref<VersionCountingBlock>> VersionCounti
return cpputils::make_unique_ref<VersionCountingBlock>(std::move(*baseBlock), std::move(dataWithHeader), blockStore);
}
inline cpputils::unique_ref<VersionCountingBlock> VersionCountingBlock::Overwrite(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, VersionCountingBlockStore *blockStore) {
uint64_t version = blockStore->knownBlockVersions()->incrementVersion(key);
cpputils::Data dataWithHeader = _prependHeaderToData(blockStore->knownBlockVersions()->myClientId(), version, std::move(data));
auto baseBlock = baseBlockStore->overwrite(key, dataWithHeader.copy()); // TODO Copy necessary?
return cpputils::make_unique_ref<VersionCountingBlock>(std::move(baseBlock), std::move(dataWithHeader), blockStore);
}
inline cpputils::Data VersionCountingBlock::_prependHeaderToData(uint32_t myClientId, uint64_t version, cpputils::Data data) {
static_assert(HEADER_LENGTH == sizeof(FORMAT_VERSION_HEADER) + sizeof(myClientId) + sizeof(version), "Wrong header length");
cpputils::Data result(data.size() + HEADER_LENGTH);
@ -140,10 +148,9 @@ inline VersionCountingBlock::VersionCountingBlock(cpputils::unique_ref<Block> ba
_blockStore(blockStore),
_baseBlock(std::move(baseBlock)),
_dataWithHeader(std::move(dataWithHeader)),
_version(_readVersion()),
_dataChanged(false),
_mutex() {
if (_version == std::numeric_limits<uint64_t>::max()) {
if (_readVersion() == std::numeric_limits<uint64_t>::max()) {
throw std::runtime_error("Version overflow when loading. This shouldn't happen because in case of a version number overflow, the block isn't stored at all.");
}
}
@ -180,10 +187,10 @@ inline void VersionCountingBlock::resize(size_t newSize) {
inline void VersionCountingBlock::_storeToBaseBlock() {
if (_dataChanged) {
_version = _blockStore->knownBlockVersions()->incrementVersion(key(), _version);
uint64_t version = _blockStore->knownBlockVersions()->incrementVersion(key());
uint32_t myClientId = _blockStore->knownBlockVersions()->myClientId();
std::memcpy(_dataWithHeader.dataOffset(CLIENTID_HEADER_OFFSET), &myClientId, sizeof(myClientId));
std::memcpy(_dataWithHeader.dataOffset(VERSION_HEADER_OFFSET), &_version, sizeof(_version));
std::memcpy(_dataWithHeader.dataOffset(VERSION_HEADER_OFFSET), &version, sizeof(version));
if (_baseBlock->size() != _dataWithHeader.size()) {
_baseBlock->resize(_dataWithHeader.size());
}

7
src/blockstore/implementations/versioncounting/VersionCountingBlockStore.cpp
View File

@ -21,7 +21,7 @@ namespace blockstore {
return _baseBlockStore->createKey();
}
optional<unique_ref<Block>> VersionCountingBlockStore::tryCreate(const Key &key, cpputils::Data data) {
optional<unique_ref<Block>> VersionCountingBlockStore::tryCreate(const Key &key, Data data) {
_checkNoPastIntegrityViolations();
//TODO Easier implementation? This is only so complicated because of the cast VersionCountingBlock -> Block
auto result = VersionCountingBlock::TryCreateNew(_baseBlockStore.get(), key, std::move(data), this);
@ -31,6 +31,11 @@ namespace blockstore {
return unique_ref<Block>(std::move(*result));
}
unique_ref<Block> VersionCountingBlockStore::overwrite(const Key &key, Data data) {
_checkNoPastIntegrityViolations();
return VersionCountingBlock::Overwrite(_baseBlockStore.get(), key, std::move(data), this);
}
optional<unique_ref<Block>> VersionCountingBlockStore::load(const Key &key) {
_checkNoPastIntegrityViolations();
auto block = _baseBlockStore->load(key);

1
src/blockstore/implementations/versioncounting/VersionCountingBlockStore.h
View File

@ -17,6 +17,7 @@ public:
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
cpputils::unique_ref<Block> overwrite(const blockstore::Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(const Key &key) override;
uint64_t numBlocks() const override;

1
src/blockstore/interface/BlockStore.h
View File

@ -20,6 +20,7 @@ public:
//TODO Use boost::optional (if key doesn't exist)
// Return nullptr if block with this key doesn't exists
virtual boost::optional<cpputils::unique_ref<Block>> load(const Key &key) = 0;
virtual cpputils::unique_ref<Block> overwrite(const blockstore::Key &key, cpputils::Data data) = 0;
virtual void remove(const Key &key) = 0;
virtual uint64_t numBlocks() const = 0;
//TODO Test estimateNumFreeBytes in all block stores

25
src/parallelaccessstore/ParallelAccessStore.h
View File

@ -53,6 +53,9 @@ public:
cpputils::unique_ref<ActualResourceRef> add(const Key &key, cpputils::unique_ref<Resource> resource, std::function<cpputils::unique_ref<ActualResourceRef>(Resource*)> createResourceRef);
boost::optional<cpputils::unique_ref<ResourceRef>> load(const Key &key);
boost::optional<cpputils::unique_ref<ResourceRef>> load(const Key &key, std::function<cpputils::unique_ref<ResourceRef>(Resource*)> createResourceRef);
//loadOrAdd: If the resource is open, run onExists() on it. If not, run onAdd() and add the created resource. Then return the resource as if load() was called on it.
cpputils::unique_ref<ResourceRef> loadOrAdd(const Key &key, std::function<void (ResourceRef*)> onExists, std::function<cpputils::unique_ref<Resource> ()> onAdd);
cpputils::unique_ref<ResourceRef> loadOrAdd(const Key &key, std::function<void (ResourceRef*)> onExists, std::function<cpputils::unique_ref<Resource> ()> onAdd, std::function<cpputils::unique_ref<ResourceRef>(Resource*)> createResourceRef);
void remove(const Key &key, cpputils::unique_ref<ResourceRef> block);
void remove(const Key &key);
@ -144,6 +147,28 @@ cpputils::unique_ref<ActualResourceRef> ParallelAccessStore<Resource, ResourceRe
return resourceRef;
}
template<class Resource, class ResourceRef, class Key>
cpputils::unique_ref<ResourceRef> ParallelAccessStore<Resource, ResourceRef, Key>::loadOrAdd(const Key &key, std::function<void (ResourceRef*)> onExists, std::function<cpputils::unique_ref<Resource> ()> onAdd) {
return loadOrAdd(key, onExists, onAdd, [] (Resource *res) {
return cpputils::make_unique_ref<ResourceRef>(res);
});
};
template<class Resource, class ResourceRef, class Key>
cpputils::unique_ref<ResourceRef> ParallelAccessStore<Resource, ResourceRef, Key>::loadOrAdd(const Key &key, std::function<void (ResourceRef*)> onExists, std::function<cpputils::unique_ref<Resource> ()> onAdd, std::function<cpputils::unique_ref<ResourceRef>(Resource*)> createResourceRef) {
std::lock_guard<std::mutex> lock(_mutex);
auto found = _openResources.find(key);
if (found == _openResources.end()) {
auto resource = onAdd();
return _add(key, std::move(resource), createResourceRef);
} else {
auto resourceRef = createResourceRef(found->second.getReference());
resourceRef->init(this, key);
onExists(resourceRef.get());
return std::move(resourceRef);
}
};
template<class Resource, class ResourceRef, class Key>
boost::optional<cpputils::unique_ref<ResourceRef>> ParallelAccessStore<Resource, ResourceRef, Key>::load(const Key &key) {
return load(key, [] (Resource *res) {

49
test/blockstore/implementations/versioncounting/KnownBlockVersionsTest.cpp
View File

@ -70,46 +70,19 @@ TEST_F(KnownBlockVersionsTest, myClientId_isConsistent) {
EXPECT_EQ(testobj.myClientId(), testobj.myClientId());
}
TEST_F(KnownBlockVersionsTest, incrementVersion_newentry_versionzero) {
auto version = testobj.incrementVersion(key, VersionCountingBlock::VERSION_ZERO);
TEST_F(KnownBlockVersionsTest, incrementVersion_newentry) {
auto version = testobj.incrementVersion(key);
EXPECT_EQ(1u, version);
EXPECT_EQ(1u, testobj.getBlockVersion(testobj.myClientId(), key));
}
TEST_F(KnownBlockVersionsTest, incrementVersion_newentry_versionnotzero) {
auto version = testobj.incrementVersion(key, 5);
TEST_F(KnownBlockVersionsTest, incrementVersion_oldentry) {
setVersion(&testobj, testobj.myClientId(), key, 5);
auto version = testobj.incrementVersion(key);
EXPECT_EQ(6u, version);
EXPECT_EQ(6u, testobj.getBlockVersion(testobj.myClientId(), key));
}
TEST_F(KnownBlockVersionsTest, incrementVersion_oldentry_sameVersion) {
setVersion(&testobj, testobj.myClientId(), key, 5);
auto version = testobj.incrementVersion(key, 5);
EXPECT_EQ(6u, version);
EXPECT_EQ(6u, testobj.getBlockVersion(testobj.myClientId(), key));
}
TEST_F(KnownBlockVersionsTest, incrementVersion_oldentry_lowerVersion1) {
setVersion(&testobj, testobj.myClientId(), key, 5);
auto version = testobj.incrementVersion(key, 4);
EXPECT_EQ(6u, version);
EXPECT_EQ(6u, testobj.getBlockVersion(testobj.myClientId(), key));
}
TEST_F(KnownBlockVersionsTest, incrementVersion_oldentry_lowerVersion2) {
setVersion(&testobj, testobj.myClientId(), key, 5);
auto version = testobj.incrementVersion(key, 3);
EXPECT_EQ(6u, version);
EXPECT_EQ(6u, testobj.getBlockVersion(testobj.myClientId(), key));
}
TEST_F(KnownBlockVersionsTest, incrementVersion_oldentry_higherVersion) {
setVersion(&testobj, testobj.myClientId(), key, 5);
auto version = testobj.incrementVersion(key, 6);
EXPECT_EQ(7u, version);
EXPECT_EQ(7u, testobj.getBlockVersion(testobj.myClientId(), key));
}
TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_newentry) {
EXPECT_TRUE(testobj.checkAndUpdateVersion(clientId, key, 5));
EXPECT_EQ(5u, testobj.getBlockVersion(clientId, key));
@ -179,7 +152,7 @@ TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientHigherVer
}
TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientLowerVersion_oldClientIsSelf) {
testobj.incrementVersion(key, 4);
setVersion(&testobj, testobj.myClientId(), key, 5);
EXPECT_TRUE(testobj.checkAndUpdateVersion(clientId2, key, 7));
EXPECT_FALSE(testobj.checkAndUpdateVersion(testobj.myClientId(), key, 3));
EXPECT_EQ(5u, testobj.getBlockVersion(testobj.myClientId(), key));
@ -187,7 +160,7 @@ TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientLowerVers
}
TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientSameVersion_oldClientIsSelf) {
testobj.incrementVersion(key, 4);
setVersion(&testobj, testobj.myClientId(), key, 5);
EXPECT_TRUE(testobj.checkAndUpdateVersion(clientId2, key, 7));
EXPECT_FALSE(testobj.checkAndUpdateVersion(testobj.myClientId(), key, 5)); // Don't allow rollback to old client's newest block, if it was superseded by another client
EXPECT_EQ(5u, testobj.getBlockVersion(testobj.myClientId(), key));
@ -195,7 +168,7 @@ TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientSameVersi
}
TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientHigherVersion_oldClientIsSelf) {
testobj.incrementVersion(key, 4);
setVersion(&testobj, testobj.myClientId(), key, 4);
EXPECT_TRUE(testobj.checkAndUpdateVersion(clientId2, key, 7));
EXPECT_TRUE(testobj.checkAndUpdateVersion(testobj.myClientId(), key, 6));
EXPECT_EQ(6u, testobj.getBlockVersion(testobj.myClientId(), key));
@ -204,7 +177,7 @@ TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientHigherVer
TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientLowerVersion_newClientIsSelf) {
setVersion(&testobj, clientId, key, 5);
testobj.incrementVersion(key, 6);
setVersion(&testobj, testobj.myClientId(), key, 7);
EXPECT_FALSE(testobj.checkAndUpdateVersion(clientId, key, 3));
EXPECT_EQ(5u, testobj.getBlockVersion(clientId, key));
EXPECT_EQ(7u, testobj.getBlockVersion(testobj.myClientId(), key));
@ -212,7 +185,7 @@ TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientLowerVers
TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientSameVersion_newClientIsSelf) {
setVersion(&testobj, clientId, key, 5);
testobj.incrementVersion(key, 6);
setVersion(&testobj, testobj.myClientId(), key, 7);
EXPECT_FALSE(testobj.checkAndUpdateVersion(clientId, key, 5)); // Don't allow rollback to old client's newest block, if it was superseded by another client
EXPECT_EQ(5u, testobj.getBlockVersion(clientId, key));
EXPECT_EQ(7u, testobj.getBlockVersion(testobj.myClientId(), key));
@ -220,7 +193,7 @@ TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientSameVersi
TEST_F(KnownBlockVersionsTest, checkAndUpdateVersion_oldentry_oldClientHigherVersion_newClientIsSelf) {
setVersion(&testobj, clientId, key, 5);
testobj.incrementVersion(key, 6);
setVersion(&testobj, testobj.myClientId(), key, 7);
EXPECT_TRUE(testobj.checkAndUpdateVersion(clientId, key, 6));
EXPECT_EQ(6u, testobj.getBlockVersion(clientId, key));
EXPECT_EQ(7u, testobj.getBlockVersion(testobj.myClientId(), key));

4
test/blockstore/interface/helpers/BlockStoreWithRandomKeysTest.cpp
View File

@ -24,6 +24,10 @@ public:
return cpputils::nullcheck(std::unique_ptr<Block>(do_create(key, data)));
}
MOCK_METHOD2(do_create, Block*(const Key &, const Data &data));
unique_ref<Block> overwrite(const Key &key, Data data) {
return cpputils::nullcheck(std::unique_ptr<Block>(do_overwrite(key, data))).value();
}
MOCK_METHOD2(do_overwrite, Block*(const Key &, const Data &data));
optional<unique_ref<Block>> load(const Key &key) {
return cpputils::nullcheck(std::unique_ptr<Block>(do_load(key)));
}

10
test/blockstore/testutils/BlockStoreTest.h
View File

@ -318,7 +318,15 @@ REGISTER_TYPED_TEST_CASE_P(BlockStoreTest,
NumBlocksIsCorrectAfterRemovingABlock_DeleteByKey,
WriteAndReadImmediately,
WriteAndReadAfterLoading,
OverwriteAndRead,
WriteTwiceAndRead,
OverwriteSameSizeAndReadImmediately,
OverwriteSameSizeAndReadAfterLoading,
OverwriteSmallerSizeAndReadImmediately,
OverwriteSmallerSizeAndReadAfterLoading,
OverwriteLargerSizeAndReadAfterLoading,
OverwriteLargerSizeAndReadImmediately,
OverwriteNonexistingAndReadAfterLoading,
OverwriteNonexistingAndReadImmediately,
CanRemoveModifiedBlock,
ForEachBlock_zeroblocks,
ForEachBlock_oneblock,

72
test/blockstore/testutils/BlockStoreTest_Data.h
View File

@ -35,7 +35,7 @@ public:
EXPECT_DATA_IS_ZEROES_OUTSIDE_OF(*loaded_block, testData.offset, testData.count);
}
void TestOverwriteAndRead() {
void TestWriteTwiceAndRead() {
auto block = blockStore->create(cpputils::Data(testData.blocksize));
block->write(backgroundData.data(), 0, testData.blocksize);
block->write(foregroundData.data(), testData.offset, testData.count);
@ -43,6 +43,66 @@ public:
EXPECT_DATA_READS_AS_OUTSIDE_OF(backgroundData, *block, testData.offset, testData.count);
}
void TestOverwriteSameSizeAndReadImmediately() {
auto key = blockStore->create(cpputils::Data(testData.blocksize))->key();
auto block = blockStore->overwrite(key, backgroundData.copy());
EXPECT_EQ(testData.blocksize, block->size());
EXPECT_DATA_READS_AS(backgroundData, *block, 0, testData.blocksize);
}
void TestOverwriteSameSizeAndReadAfterLoading() {
auto key = blockStore->create(cpputils::Data(testData.blocksize))->key();
blockStore->overwrite(key, backgroundData.copy());
auto block = blockStore->load(key).value();
EXPECT_EQ(testData.blocksize, block->size());
EXPECT_DATA_READS_AS(backgroundData, *block, 0, testData.blocksize);
}
void TestOverwriteSmallerSizeAndReadImmediately() {
auto key = blockStore->create(cpputils::Data(testData.blocksize))->key();
auto block = blockStore->overwrite(key, foregroundData.copy());
EXPECT_EQ(testData.count, block->size());
EXPECT_DATA_READS_AS(foregroundData, *block, 0, testData.count);
}
void TestOverwriteSmallerSizeAndReadAfterLoading() {
auto key = blockStore->create(cpputils::Data(testData.blocksize))->key();
blockStore->overwrite(key, foregroundData.copy());
auto block = blockStore->load(key).value();
EXPECT_EQ(testData.count, block->size());
EXPECT_DATA_READS_AS(foregroundData, *block, 0, testData.count);
}
void TestOverwriteLargerSizeAndReadImmediately() {
auto key = blockStore->create(cpputils::Data(testData.count))->key();
auto block = blockStore->overwrite(key, backgroundData.copy());
EXPECT_EQ(testData.blocksize, block->size());
EXPECT_DATA_READS_AS(backgroundData, *block, 0, testData.blocksize);
}
void TestOverwriteLargerSizeAndReadAfterLoading() {
auto key = blockStore->create(cpputils::Data(testData.count))->key();
blockStore->overwrite(key, backgroundData.copy());
auto block = blockStore->load(key).value();
EXPECT_EQ(testData.blocksize, block->size());
EXPECT_DATA_READS_AS(backgroundData, *block, 0, testData.blocksize);
}
void TestOverwriteNonexistingAndReadImmediately() {
auto key = blockStore->createKey();
auto block = blockStore->overwrite(key, backgroundData.copy());
EXPECT_EQ(testData.blocksize, block->size());
EXPECT_DATA_READS_AS(backgroundData, *block, 0, testData.blocksize);
}
void TestOverwriteNonexistingAndReadAfterLoading() {
auto key = blockStore->createKey();
blockStore->overwrite(key, backgroundData.copy());
auto block = blockStore->load(key).value();
EXPECT_EQ(testData.blocksize, block->size());
EXPECT_DATA_READS_AS(backgroundData, *block, 0, testData.blocksize);
}
private:
cpputils::unique_ref<blockstore::BlockStore> blockStore;
DataRange testData;
@ -102,6 +162,14 @@ inline std::vector<DataRange> DATA_RANGES() {
TYPED_TEST_P_FOR_ALL_DATA_RANGES(WriteAndReadImmediately);
TYPED_TEST_P_FOR_ALL_DATA_RANGES(WriteAndReadAfterLoading);
TYPED_TEST_P_FOR_ALL_DATA_RANGES(OverwriteAndRead);
TYPED_TEST_P_FOR_ALL_DATA_RANGES(WriteTwiceAndRead);
TYPED_TEST_P_FOR_ALL_DATA_RANGES(OverwriteSameSizeAndReadImmediately);
TYPED_TEST_P_FOR_ALL_DATA_RANGES(OverwriteSameSizeAndReadAfterLoading);
TYPED_TEST_P_FOR_ALL_DATA_RANGES(OverwriteSmallerSizeAndReadImmediately);
TYPED_TEST_P_FOR_ALL_DATA_RANGES(OverwriteSmallerSizeAndReadAfterLoading);
TYPED_TEST_P_FOR_ALL_DATA_RANGES(OverwriteLargerSizeAndReadImmediately);
TYPED_TEST_P_FOR_ALL_DATA_RANGES(OverwriteLargerSizeAndReadAfterLoading);
TYPED_TEST_P_FOR_ALL_DATA_RANGES(OverwriteNonexistingAndReadImmediately);
TYPED_TEST_P_FOR_ALL_DATA_RANGES(OverwriteNonexistingAndReadAfterLoading);
#endif