Introduced ParallelAccessFsBlobStore to avoid race conditions when accessing the same FsBlob in parallel

This commit is contained in:
Sebastian Messmer 2015-10-04 17:20:14 +02:00
parent 37f7c764d1
commit 1977a720df
35 changed files with 475 additions and 135 deletions

View File

@ -8,6 +8,7 @@
messmer/cpp-utils: 2
messmer/fspp: 0
messmer/gitversion: 5
messmer/parallelaccessstore: 0
[parent]
messmer/cryfs: 0

View File

@ -1,6 +1,6 @@
#include <messmer/blockstore/implementations/caching/CachingBlockStore.h>
#include <messmer/blockstore/implementations/encrypted/ciphers/ciphers.h>
#include "fsblobstore/DirBlob.h"
#include "parallelaccessfsblobstore/DirBlobRef.h"
#include "CryDevice.h"
#include "CryDir.h"
@ -11,7 +11,7 @@
#include "messmer/blobstore/implementations/onblocks/BlobStoreOnBlocks.h"
#include "messmer/blobstore/implementations/onblocks/BlobOnBlocks.h"
#include "messmer/blockstore/implementations/encrypted/EncryptedBlockStore.h"
#include "fsblobstore/FsBlobStore.h"
#include "parallelaccessfsblobstore/ParallelAccessFsBlobStore.h"
using std::string;
@ -32,10 +32,11 @@ using cpputils::dynamic_pointer_move;
using boost::optional;
using boost::none;
using cryfs::fsblobstore::FsBlobStore;
using cryfs::fsblobstore::FileBlob;
using cryfs::fsblobstore::DirBlob;
using cryfs::fsblobstore::SymlinkBlob;
using cryfs::fsblobstore::FsBlob;
using cryfs::parallelaccessfsblobstore::ParallelAccessFsBlobStore;
using cryfs::parallelaccessfsblobstore::FileBlobRef;
using cryfs::parallelaccessfsblobstore::DirBlobRef;
using cryfs::parallelaccessfsblobstore::SymlinkBlobRef;
using cryfs::parallelaccessfsblobstore::FsBlobRef;
namespace bf = boost::filesystem;
@ -44,11 +45,14 @@ namespace cryfs {
constexpr uint32_t CryDevice::BLOCKSIZE_BYTES;
CryDevice::CryDevice(unique_ref<CryConfig> config, unique_ref<BlockStore> blockStore)
: _fsBlobStore(make_unique_ref<FsBlobStore>(
make_unique_ref<BlobStoreOnBlocks>(
make_unique_ref<CachingBlockStore>(
CreateEncryptedBlockStore(*config, std::move(blockStore))
), BLOCKSIZE_BYTES))),
: _fsBlobStore(
make_unique_ref<ParallelAccessFsBlobStore>(
make_unique_ref<FsBlobStore>(
make_unique_ref<BlobStoreOnBlocks>(
make_unique_ref<CachingBlockStore>(
CreateEncryptedBlockStore(*config, std::move(blockStore))
), BLOCKSIZE_BYTES)))
),
_rootKey(GetOrCreateRootKey(config.get())) {
}
@ -80,21 +84,21 @@ optional<unique_ref<fspp::Node>> CryDevice::Load(const bf::path &path) {
}
}
unique_ref<DirBlob> CryDevice::LoadDirBlob(const bf::path &path) {
unique_ref<DirBlobRef> CryDevice::LoadDirBlob(const bf::path &path) {
auto blob = LoadBlob(path);
auto dir = dynamic_pointer_move<DirBlob>(blob);
auto dir = dynamic_pointer_move<DirBlobRef>(blob);
if (dir == none) {
throw FuseErrnoException(ENOTDIR); // Loaded blob is not a directory
}
return std::move(*dir);
}
unique_ref<FsBlob> CryDevice::LoadBlob(const bf::path &path) {
unique_ref<FsBlobRef> CryDevice::LoadBlob(const bf::path &path) {
auto currentBlob = _fsBlobStore->load(_rootKey);
ASSERT(currentBlob != none, "rootDir not found");
for (const bf::path &component : path.relative_path()) {
auto currentDir = dynamic_pointer_move<DirBlob>(*currentBlob);
auto currentDir = dynamic_pointer_move<DirBlobRef>(*currentBlob);
if (currentDir == none) {
throw FuseErrnoException(ENOTDIR); // Path component is not a dir
}
@ -117,19 +121,19 @@ void CryDevice::statfs(const bf::path &path, struct statvfs *fsstat) {
throw FuseErrnoException(ENOTSUP);
}
unique_ref<FileBlob> CryDevice::CreateFileBlob() {
unique_ref<FileBlobRef> CryDevice::CreateFileBlob() {
return _fsBlobStore->createFileBlob();
}
unique_ref<DirBlob> CryDevice::CreateDirBlob() {
unique_ref<DirBlobRef> CryDevice::CreateDirBlob() {
return _fsBlobStore->createDirBlob();
}
unique_ref<SymlinkBlob> CryDevice::CreateSymlinkBlob(const bf::path &target) {
unique_ref<SymlinkBlobRef> CryDevice::CreateSymlinkBlob(const bf::path &target) {
return _fsBlobStore->createSymlinkBlob(target);
}
unique_ref<FsBlob> CryDevice::LoadBlob(const blockstore::Key &key) {
unique_ref<FsBlobRef> CryDevice::LoadBlob(const blockstore::Key &key) {
auto blob = _fsBlobStore->load(key);
ASSERT(blob != none, "Blob not found");
return std::move(*blob);

View File

@ -8,10 +8,10 @@
#include <boost/filesystem.hpp>
#include <messmer/fspp/fs_interface/Device.h>
#include "fsblobstore/FsBlobStore.h"
#include "fsblobstore/DirBlob.h"
#include "fsblobstore/FileBlob.h"
#include "fsblobstore/SymlinkBlob.h"
#include "parallelaccessfsblobstore/ParallelAccessFsBlobStore.h"
#include "parallelaccessfsblobstore/DirBlobRef.h"
#include "parallelaccessfsblobstore/FileBlobRef.h"
#include "parallelaccessfsblobstore/SymlinkBlobRef.h"
namespace cryfs {
@ -24,12 +24,12 @@ public:
void statfs(const boost::filesystem::path &path, struct ::statvfs *fsstat) override;
cpputils::unique_ref<fsblobstore::FileBlob> CreateFileBlob();
cpputils::unique_ref<fsblobstore::DirBlob> CreateDirBlob();
cpputils::unique_ref<fsblobstore::SymlinkBlob> CreateSymlinkBlob(const boost::filesystem::path &target);
cpputils::unique_ref<fsblobstore::FsBlob> LoadBlob(const blockstore::Key &key); //TODO Do I still need this function?
cpputils::unique_ref<fsblobstore::FsBlob> LoadBlob(const boost::filesystem::path &path);
cpputils::unique_ref<fsblobstore::DirBlob> LoadDirBlob(const boost::filesystem::path &path);
cpputils::unique_ref<parallelaccessfsblobstore::FileBlobRef> CreateFileBlob();
cpputils::unique_ref<parallelaccessfsblobstore::DirBlobRef> CreateDirBlob();
cpputils::unique_ref<parallelaccessfsblobstore::SymlinkBlobRef> CreateSymlinkBlob(const boost::filesystem::path &target);
cpputils::unique_ref<parallelaccessfsblobstore::FsBlobRef> LoadBlob(const blockstore::Key &key); //TODO Do I still need this function?
cpputils::unique_ref<parallelaccessfsblobstore::FsBlobRef> LoadBlob(const boost::filesystem::path &path);
cpputils::unique_ref<parallelaccessfsblobstore::DirBlobRef> LoadDirBlob(const boost::filesystem::path &path);
void RemoveBlob(const blockstore::Key &key);
boost::optional<cpputils::unique_ref<fspp::Node>> Load(const boost::filesystem::path &path) override;
@ -37,7 +37,7 @@ public:
private:
cpputils::unique_ref<fsblobstore::FsBlobStore> _fsBlobStore;
cpputils::unique_ref<parallelaccessfsblobstore::ParallelAccessFsBlobStore> _fsBlobStore;
blockstore::Key _rootKey;

View File

@ -9,7 +9,6 @@
#include "CryDevice.h"
#include "CryFile.h"
#include "CryOpenFile.h"
#include "fsblobstore/SymlinkBlob.h"
//TODO Get rid of this in favor of exception hierarchy
using fspp::fuse::CHECK_RETVAL;
@ -26,11 +25,11 @@ using cpputils::make_unique_ref;
using cpputils::dynamic_pointer_move;
using boost::optional;
using boost::none;
using cryfs::fsblobstore::DirBlob;
using cryfs::parallelaccessfsblobstore::DirBlobRef;
namespace cryfs {
CryDir::CryDir(CryDevice *device, boost::optional<unique_ref<DirBlob>> parent, const Key &key)
CryDir::CryDir(CryDevice *device, boost::optional<unique_ref<DirBlobRef>> parent, const Key &key)
: CryNode(device, std::move(parent), key) {
}
@ -38,9 +37,8 @@ CryDir::~CryDir() {
}
unique_ref<fspp::OpenFile> CryDir::createAndOpenFile(const string &name, mode_t mode, uid_t uid, gid_t gid) {
auto blob = LoadBlob();
auto child = device()->CreateFileBlob();
blob->AddChildFile(name, child->key(), mode, uid, gid);
LoadBlob()->AddChildFile(name, child->key(), mode, uid, gid);
return make_unique_ref<CryOpenFile>(std::move(child));
}
@ -50,9 +48,9 @@ void CryDir::createDir(const string &name, mode_t mode, uid_t uid, gid_t gid) {
blob->AddChildDir(name, child->key(), mode, uid, gid);
}
unique_ref<DirBlob> CryDir::LoadBlob() const {
unique_ref<DirBlobRef> CryDir::LoadBlob() const {
auto blob = CryNode::LoadBlob();
auto dir_blob = dynamic_pointer_move<DirBlob>(blob);
auto dir_blob = dynamic_pointer_move<DirBlobRef>(blob);
ASSERT(dir_blob != none, "Blob does not store a directory");
return std::move(*dir_blob);
}

View File

@ -4,13 +4,13 @@
#include <messmer/fspp/fs_interface/Dir.h>
#include "CryNode.h"
#include "fsblobstore/DirBlob.h"
#include "parallelaccessfsblobstore/DirBlobRef.h"
namespace cryfs {
class CryDir: public fspp::Dir, CryNode {
public:
CryDir(CryDevice *device, boost::optional<cpputils::unique_ref<fsblobstore::DirBlob>> parent, const blockstore::Key &key);
CryDir(CryDevice *device, boost::optional<cpputils::unique_ref<parallelaccessfsblobstore::DirBlobRef>> parent, const blockstore::Key &key);
virtual ~CryDir();
//TODO return type variance to CryFile/CryDir?
@ -24,7 +24,7 @@ public:
fspp::Dir::EntryType getType() const override;
private:
cpputils::unique_ref<fsblobstore::DirBlob> LoadBlob() const;
cpputils::unique_ref<parallelaccessfsblobstore::DirBlobRef> LoadBlob() const;
DISALLOW_COPY_AND_ASSIGN(CryDir);
};

View File

@ -15,21 +15,21 @@ using boost::none;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using cpputils::dynamic_pointer_move;
using cryfs::fsblobstore::DirBlob;
using cryfs::fsblobstore::FileBlob;
using cryfs::parallelaccessfsblobstore::DirBlobRef;
using cryfs::parallelaccessfsblobstore::FileBlobRef;
namespace cryfs {
CryFile::CryFile(CryDevice *device, unique_ref<DirBlob> parent, const Key &key)
CryFile::CryFile(CryDevice *device, unique_ref<DirBlobRef> parent, const Key &key)
: CryNode(device, std::move(parent), key) {
}
CryFile::~CryFile() {
}
unique_ref<fsblobstore::FileBlob> CryFile::LoadBlob() const {
unique_ref<parallelaccessfsblobstore::FileBlobRef> CryFile::LoadBlob() const {
auto blob = CryNode::LoadBlob();
auto file_blob = dynamic_pointer_move<FileBlob>(blob);
auto file_blob = dynamic_pointer_move<FileBlobRef>(blob);
ASSERT(file_blob != none, "Blob does not store a file");
return std::move(*file_blob);
}

View File

@ -2,8 +2,8 @@
#ifndef CRYFS_LIB_CRYFILE_H_
#define CRYFS_LIB_CRYFILE_H_
#include "fsblobstore/FileBlob.h"
#include "fsblobstore/DirBlob.h"
#include "parallelaccessfsblobstore/FileBlobRef.h"
#include "parallelaccessfsblobstore/DirBlobRef.h"
#include <messmer/fspp/fs_interface/File.h>
#include "CryNode.h"
@ -11,7 +11,7 @@ namespace cryfs {
class CryFile: public fspp::File, CryNode {
public:
CryFile(CryDevice *device, cpputils::unique_ref<fsblobstore::DirBlob> parent, const blockstore::Key &key);
CryFile(CryDevice *device, cpputils::unique_ref<parallelaccessfsblobstore::DirBlobRef> parent, const blockstore::Key &key);
virtual ~CryFile();
cpputils::unique_ref<fspp::OpenFile> open(int flags) const override;
@ -19,7 +19,7 @@ public:
fspp::Dir::EntryType getType() const override;
private:
cpputils::unique_ref<fsblobstore::FileBlob> LoadBlob() const;
cpputils::unique_ref<parallelaccessfsblobstore::FileBlobRef> LoadBlob() const;
DISALLOW_COPY_AND_ASSIGN(CryFile);
};

View File

@ -16,8 +16,8 @@ using cpputils::dynamic_pointer_move;
using cpputils::unique_ref;
using boost::optional;
using boost::none;
using cryfs::fsblobstore::FsBlob;
using cryfs::fsblobstore::DirBlob;
using cryfs::parallelaccessfsblobstore::FsBlobRef;
using cryfs::parallelaccessfsblobstore::DirBlobRef;
//TODO Get rid of this in favor of an exception hierarchy
using fspp::fuse::CHECK_RETVAL;
@ -25,7 +25,7 @@ using fspp::fuse::FuseErrnoException;
namespace cryfs {
CryNode::CryNode(CryDevice *device, optional<unique_ref<DirBlob>> parent, const Key &key)
CryNode::CryNode(CryDevice *device, optional<unique_ref<DirBlobRef>> parent, const Key &key)
: _device(device),
_parent(std::move(parent)),
_key(key) {
@ -82,7 +82,7 @@ const CryDevice *CryNode::device() const {
return _device;
}
unique_ref<FsBlob> CryNode::LoadBlob() const {
unique_ref<FsBlobRef> CryNode::LoadBlob() const {
return _device->LoadBlob(_key);
}

View File

@ -5,14 +5,14 @@
#include <messmer/fspp/fs_interface/Node.h>
#include "messmer/cpp-utils/macros.h"
#include <messmer/fspp/fs_interface/Dir.h>
#include "fsblobstore/DirBlob.h"
#include "parallelaccessfsblobstore/DirBlobRef.h"
#include "CryDevice.h"
namespace cryfs {
class CryNode: public virtual fspp::Node {
public:
CryNode(CryDevice *device, boost::optional<cpputils::unique_ref<fsblobstore::DirBlob>> parent, const blockstore::Key &key);
CryNode(CryDevice *device, boost::optional<cpputils::unique_ref<parallelaccessfsblobstore::DirBlobRef>> parent, const blockstore::Key &key);
void access(int mask) const override;
void stat(struct ::stat *result) const override;
void chmod(mode_t mode) override;
@ -27,13 +27,13 @@ protected:
CryDevice *device();
const CryDevice *device() const;
cpputils::unique_ref<fsblobstore::FsBlob> LoadBlob() const;
cpputils::unique_ref<parallelaccessfsblobstore::FsBlobRef> LoadBlob() const;
virtual fspp::Dir::EntryType getType() const = 0;
private:
CryDevice *_device;
boost::optional<cpputils::unique_ref<fsblobstore::DirBlob>> _parent;
boost::optional<cpputils::unique_ref<parallelaccessfsblobstore::DirBlobRef>> _parent;
blockstore::Key _key;
DISALLOW_COPY_AND_ASSIGN(CryNode);

View File

@ -9,7 +9,7 @@
namespace bf = boost::filesystem;
using cpputils::unique_ref;
using cryfs::fsblobstore::FileBlob;
using cryfs::parallelaccessfsblobstore::FileBlobRef;
//TODO Get rid of this in favor of a exception hierarchy
using fspp::fuse::CHECK_RETVAL;
@ -17,7 +17,7 @@ using fspp::fuse::FuseErrnoException;
namespace cryfs {
CryOpenFile::CryOpenFile(unique_ref<FileBlob> fileBlob)
CryOpenFile::CryOpenFile(unique_ref<FileBlobRef> fileBlob)
: _fileBlob(std::move(fileBlob)) {
}

View File

@ -3,14 +3,14 @@
#define CRYFS_LIB_CRYOPENFILE_H_
#include "messmer/fspp/fs_interface/OpenFile.h"
#include "fsblobstore/FileBlob.h"
#include "parallelaccessfsblobstore/FileBlobRef.h"
namespace cryfs {
class CryDevice;
class CryOpenFile: public fspp::OpenFile {
public:
explicit CryOpenFile(cpputils::unique_ref<fsblobstore::FileBlob> fileBlob);
explicit CryOpenFile(cpputils::unique_ref<parallelaccessfsblobstore::FileBlobRef> fileBlob);
virtual ~CryOpenFile();
void stat(struct ::stat *result) const override;
@ -22,7 +22,7 @@ public:
void fdatasync() override;
private:
cpputils::unique_ref<fsblobstore::FileBlob> _fileBlob;
cpputils::unique_ref<parallelaccessfsblobstore::FileBlobRef> _fileBlob;
DISALLOW_COPY_AND_ASSIGN(CryOpenFile);
};

View File

@ -3,7 +3,7 @@
#include "messmer/fspp/fuse/FuseErrnoException.h"
#include "CryDevice.h"
#include "CrySymlink.h"
#include "fsblobstore/SymlinkBlob.h"
#include "parallelaccessfsblobstore/SymlinkBlobRef.h"
//TODO Get rid of this in favor of exception hierarchy
using fspp::fuse::CHECK_RETVAL;
@ -20,21 +20,21 @@ using boost::optional;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using cpputils::dynamic_pointer_move;
using cryfs::fsblobstore::SymlinkBlob;
using cryfs::fsblobstore::DirBlob;
using cryfs::parallelaccessfsblobstore::SymlinkBlobRef;
using cryfs::parallelaccessfsblobstore::DirBlobRef;
namespace cryfs {
CrySymlink::CrySymlink(CryDevice *device, unique_ref<DirBlob> parent, const Key &key)
CrySymlink::CrySymlink(CryDevice *device, unique_ref<DirBlobRef> parent, const Key &key)
: CryNode(device, std::move(parent), key) {
}
CrySymlink::~CrySymlink() {
}
unique_ref<SymlinkBlob> CrySymlink::LoadBlob() const {
unique_ref<SymlinkBlobRef> CrySymlink::LoadBlob() const {
auto blob = CryNode::LoadBlob();
auto symlink_blob = dynamic_pointer_move<SymlinkBlob>(blob);
auto symlink_blob = dynamic_pointer_move<SymlinkBlobRef>(blob);
ASSERT(symlink_blob != none, "Blob does not store a symlink");
return std::move(*symlink_blob);
}

View File

@ -4,14 +4,14 @@
#include <messmer/fspp/fs_interface/Symlink.h>
#include "CryNode.h"
#include "fsblobstore/SymlinkBlob.h"
#include "fsblobstore/DirBlob.h"
#include "parallelaccessfsblobstore/SymlinkBlobRef.h"
#include "parallelaccessfsblobstore/DirBlobRef.h"
namespace cryfs {
class CrySymlink: public fspp::Symlink, CryNode {
public:
CrySymlink(CryDevice *device, cpputils::unique_ref<fsblobstore::DirBlob> parent, const blockstore::Key &key);
CrySymlink(CryDevice *device, cpputils::unique_ref<parallelaccessfsblobstore::DirBlobRef> parent, const blockstore::Key &key);
virtual ~CrySymlink();
boost::filesystem::path target() const override;
@ -19,7 +19,7 @@ public:
fspp::Dir::EntryType getType() const override;
private:
cpputils::unique_ref<fsblobstore::SymlinkBlob> LoadBlob() const;
cpputils::unique_ref<parallelaccessfsblobstore::SymlinkBlobRef> LoadBlob() const;
DISALLOW_COPY_AND_ASSIGN(CrySymlink);
};

View File

@ -26,10 +26,10 @@ using boost::none;
namespace cryfs {
namespace fsblobstore {
//TODO Factor out a DirEntryList class
//TODO Factor out a DirEntryList class
DirBlob::DirBlob(unique_ref<Blob> blob, FsBlobStore *fsBlobStore, std::function<void()> onDestruct) :
FsBlob(std::move(blob), onDestruct), _fsBlobStore(fsBlobStore), _entries(), _changed(false) {
DirBlob::DirBlob(unique_ref<Blob> blob, std::function<off_t (const blockstore::Key&)> getLstatSize) :
FsBlob(std::move(blob)), _getLstatSize(getLstatSize), _entries(), _changed(false) {
ASSERT(magicNumber() == MagicNumbers::DIR, "Loaded blob is not a directory");
_readEntriesFromBlob();
}
@ -43,9 +43,9 @@ void DirBlob::flush() {
baseBlob().flush();
}
unique_ref<DirBlob> DirBlob::InitializeEmptyDir(unique_ref<Blob> blob, FsBlobStore *fsBlobStore, std::function<void()> onDestruct) {
unique_ref<DirBlob> DirBlob::InitializeEmptyDir(unique_ref<Blob> blob, std::function<off_t(const blockstore::Key&)> getLstatSize) {
InitializeBlobWithMagicNumber(blob.get(), MagicNumbers::DIR);
return make_unique_ref<DirBlob>(std::move(blob), fsBlobStore, onDestruct);
return make_unique_ref<DirBlob>(std::move(blob), getLstatSize);
}
void DirBlob::_writeEntriesToBlob() {
@ -205,12 +205,10 @@ void DirBlob::statChild(const Key &key, struct ::stat *result) const {
result->st_nlink = 1;
//TODO Handle file access times
result->st_mtime = result->st_ctime = result->st_atime = 0;
auto blob = _fsBlobStore->load(key);
ASSERT(blob != none, "Blob for directory entry not found");
result->st_size = (*blob)->lstat_size();
result->st_size = _getLstatSize(key);
//TODO Move ceilDivision to general utils which can be used by cryfs as well
result->st_blocks = blobstore::onblocks::utils::ceilDivision(result->st_size, 512);
result->st_blksize = CryDevice::BLOCKSIZE_BYTES; //TODO _fsBlobStore->BLOCKSIZE_BYTES would be cleaner
result->st_blksize = CryDevice::BLOCKSIZE_BYTES; //TODO FsBlobStore::BLOCKSIZE_BYTES would be cleaner
}
void DirBlob::chmodChild(const Key &key, mode_t mode) {
@ -232,5 +230,9 @@ void DirBlob::chownChild(const Key &key, uid_t uid, gid_t gid) {
}
}
void DirBlob::setLstatSizeGetter(std::function<off_t(const blockstore::Key&)> getLstatSize) {
_getLstatSize = getLstatSize;
}
}
}

View File

@ -6,7 +6,6 @@
#include <messmer/cpp-utils/macros.h>
#include <messmer/fspp/fs_interface/Dir.h>
#include "FsBlob.h"
#include <vector>
namespace cryfs {
@ -43,10 +42,9 @@ namespace cryfs {
};
static cpputils::unique_ref<DirBlob> InitializeEmptyDir(cpputils::unique_ref<blobstore::Blob> blob,
FsBlobStore *fsBlobStore,
std::function<void()> onDestruct);
std::function<off_t (const blockstore::Key&)> getLstatSize);
DirBlob(cpputils::unique_ref<blobstore::Blob> blob, FsBlobStore *fsBlobStore, std::function<void()> onDestruct);
DirBlob(cpputils::unique_ref<blobstore::Blob> blob, std::function<off_t (const blockstore::Key&)> getLstatSize);
virtual ~DirBlob();
@ -80,6 +78,8 @@ namespace cryfs {
void chownChild(const blockstore::Key &key, uid_t uid, gid_t gid);
void setLstatSizeGetter(std::function<off_t(const blockstore::Key&)> getLstatSize);
private:
const char *readAndAddNextChild(const char *pos, std::vector<Entry> *result) const;
@ -92,7 +92,7 @@ namespace cryfs {
std::vector<DirBlob::Entry>::iterator _findChild(const blockstore::Key &key);
FsBlobStore *_fsBlobStore;
std::function<off_t (const blockstore::Key&)> _getLstatSize;
std::vector<Entry> _entries;
bool _changed;

View File

@ -7,17 +7,18 @@
using blobstore::Blob;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using blockstore::Key;
namespace cryfs {
namespace fsblobstore {
FileBlob::FileBlob(unique_ref<Blob> blob, std::function<void()> onDestruct)
: FsBlob(std::move(blob), onDestruct) {
FileBlob::FileBlob(unique_ref<Blob> blob)
: FsBlob(std::move(blob)) {
}
unique_ref<FileBlob> FileBlob::InitializeEmptyFile(unique_ref<Blob> blob, std::function<void()> onDestruct) {
unique_ref<FileBlob> FileBlob::InitializeEmptyFile(unique_ref<Blob> blob) {
InitializeBlobWithMagicNumber(blob.get(), MagicNumbers::FILE);
return make_unique_ref<FileBlob>(std::move(blob), onDestruct);
return make_unique_ref<FileBlob>(std::move(blob));
}
ssize_t FileBlob::read(void *target, uint64_t offset, uint64_t count) const {
@ -32,10 +33,6 @@ void FileBlob::flush() {
baseBlob().flush();
}
blockstore::Key FileBlob::key() const {
return baseBlob().key();
}
void FileBlob::resize(off_t size) {
baseBlob().resize(size+1);
}

View File

@ -9,9 +9,9 @@ namespace cryfs {
class FileBlob: public FsBlob {
public:
static cpputils::unique_ref<FileBlob> InitializeEmptyFile(cpputils::unique_ref<blobstore::Blob> blob, std::function<void()> onDestruct);
static cpputils::unique_ref<FileBlob> InitializeEmptyFile(cpputils::unique_ref<blobstore::Blob> blob);
FileBlob(cpputils::unique_ref<blobstore::Blob> blob, std::function<void()> onDestruct);
FileBlob(cpputils::unique_ref<blobstore::Blob> blob);
ssize_t read(void *target, uint64_t offset, uint64_t count) const;
@ -25,8 +25,6 @@ namespace cryfs {
off_t size() const;
blockstore::Key key() const;
};
}
}

View File

@ -3,7 +3,6 @@
#include <messmer/cpp-utils/pointer/unique_ref.h>
#include <messmer/blobstore/interface/Blob.h>
#include <functional>
namespace cryfs {
namespace fsblobstore {
@ -12,10 +11,10 @@ namespace cryfs {
virtual ~FsBlob();
virtual off_t lstat_size() const = 0;
blockstore::Key key() const;
const blockstore::Key &key() const;
protected:
FsBlob(cpputils::unique_ref<blobstore::Blob> baseBlob, std::function<void()> onDestruct);
FsBlob(cpputils::unique_ref<blobstore::Blob> baseBlob);
blobstore::Blob &baseBlob();
const blobstore::Blob &baseBlob() const;
@ -30,20 +29,18 @@ namespace cryfs {
cpputils::unique_ref<blobstore::Blob> releaseBaseBlob();
cpputils::unique_ref<blobstore::Blob> _baseBlob;
std::function<void()> _onDestruct;
DISALLOW_COPY_AND_ASSIGN(FsBlob);
};
inline FsBlob::FsBlob(cpputils::unique_ref<blobstore::Blob> baseBlob, std::function<void()> onDestruct)
: _baseBlob(std::move(baseBlob)), _onDestruct(onDestruct) {
inline FsBlob::FsBlob(cpputils::unique_ref<blobstore::Blob> baseBlob)
: _baseBlob(std::move(baseBlob)) {
}
inline FsBlob::~FsBlob() {
_onDestruct();
}
inline blockstore::Key FsBlob::key() const {
inline const blockstore::Key &FsBlob::key() const {
return _baseBlob->key();
}

View File

@ -8,6 +8,7 @@ namespace bf = boost::filesystem;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using blobstore::BlobStore;
using blockstore::Key;
using boost::none;
using std::function;
@ -19,41 +20,31 @@ FsBlobStore::FsBlobStore(unique_ref<BlobStore> baseBlobStore): _baseBlobStore(st
unique_ref<FileBlob> FsBlobStore::createFileBlob() {
auto blob = _baseBlobStore->create();
auto key = blob->key();
_openBlobs.lock(key);
return FileBlob::InitializeEmptyFile(std::move(blob), freeLockFunction(key));
return FileBlob::InitializeEmptyFile(std::move(blob));
}
unique_ref<DirBlob> FsBlobStore::createDirBlob() {
auto blob = _baseBlobStore->create();
auto key = blob->key();
_openBlobs.lock(key);
//TODO Passed in fsBlobStore should be ParallelAccessFsBlobStore later
return DirBlob::InitializeEmptyDir(std::move(blob), this, freeLockFunction(key));
return DirBlob::InitializeEmptyDir(std::move(blob), _getLstatSize());
}
unique_ref<SymlinkBlob> FsBlobStore::createSymlinkBlob(const bf::path &target) {
auto blob = _baseBlobStore->create();
auto key = blob->key();
_openBlobs.lock(key);
return SymlinkBlob::InitializeSymlink(std::move(blob), target, freeLockFunction(key));
return SymlinkBlob::InitializeSymlink(std::move(blob), target);
}
boost::optional<unique_ref<FsBlob>> FsBlobStore::load(const blockstore::Key &key) {
_openBlobs.lock(key);
auto blob = _baseBlobStore->load(key);
if (blob == none) {
return none;
}
unsigned char magicNumber = FsBlob::magicNumber(**blob);
if (magicNumber == MagicNumbers::FILE) {
return unique_ref<FsBlob>(make_unique_ref<FileBlob>(std::move(*blob), freeLockFunction(key)));
return unique_ref<FsBlob>(make_unique_ref<FileBlob>(std::move(*blob)));
} else if (magicNumber == MagicNumbers::DIR) {
//TODO Passed in fsBlobStore should be ParallelAccessFsBlobStore later
return unique_ref<FsBlob>(make_unique_ref<DirBlob>(std::move(*blob), this, freeLockFunction(key)));
return unique_ref<FsBlob>(make_unique_ref<DirBlob>(std::move(*blob), _getLstatSize()));
} else if (magicNumber == MagicNumbers::SYMLINK) {
return unique_ref<FsBlob>(make_unique_ref<SymlinkBlob>(std::move(*blob), freeLockFunction(key)));
return unique_ref<FsBlob>(make_unique_ref<SymlinkBlob>(std::move(*blob)));
} else {
ASSERT(false, "Unknown magic number");
}
@ -63,9 +54,11 @@ void FsBlobStore::remove(cpputils::unique_ref<FsBlob> blob) {
_baseBlobStore->remove(blob->releaseBaseBlob());
}
function<void()> FsBlobStore::freeLockFunction(const blockstore::Key &key) {
return [this, key] {
_openBlobs.release(key);
function<off_t (const Key &)> FsBlobStore::_getLstatSize() {
return [this] (const Key &key) {
auto blob = load(key);
ASSERT(blob != none, "Blob not found");
return (*blob)->lstat_size();
};
}

View File

@ -23,10 +23,9 @@ namespace cryfs {
void remove(cpputils::unique_ref<FsBlob> blob);
private:
std::function<void()> freeLockFunction(const blockstore::Key &key);
//Instead of locking open blobs, it would be faster to allow parallel access similar to parallelaccessstore.
cpputils::LockPool<blockstore::Key> _openBlobs;
std::function<off_t(const blockstore::Key &)> _getLstatSize();
cpputils::unique_ref<blobstore::BlobStore> _baseBlobStore;
};
}

View File

@ -14,17 +14,17 @@ namespace bf = boost::filesystem;
namespace cryfs {
namespace fsblobstore {
SymlinkBlob::SymlinkBlob(unique_ref<Blob> blob, std::function<void()> onDestruct)
: FsBlob(std::move(blob), onDestruct), _target(_readTargetFromBlob(baseBlob())) {
SymlinkBlob::SymlinkBlob(unique_ref<Blob> blob)
: FsBlob(std::move(blob)), _target(_readTargetFromBlob(baseBlob())) {
}
unique_ref<SymlinkBlob> SymlinkBlob::InitializeSymlink(unique_ref<Blob> blob, const bf::path &target, std::function<void()> onDestruct) {
unique_ref<SymlinkBlob> SymlinkBlob::InitializeSymlink(unique_ref<Blob> blob, const bf::path &target) {
string targetStr = target.native();
blob->resize(1 + targetStr.size());
unsigned char magicNumber = MagicNumbers::SYMLINK;
blob->write(&magicNumber, 0, 1);
blob->write(targetStr.c_str(), 1, targetStr.size());
return make_unique_ref<SymlinkBlob>(std::move(blob), onDestruct);
return make_unique_ref<SymlinkBlob>(std::move(blob));
}
void SymlinkBlob::_checkMagicNumber(const Blob &blob) {

View File

@ -11,10 +11,9 @@ namespace cryfs {
class SymlinkBlob: public FsBlob {
public:
static cpputils::unique_ref<SymlinkBlob> InitializeSymlink(cpputils::unique_ref<blobstore::Blob> blob,
const boost::filesystem::path &target,
std::function<void()> onDestruct);
const boost::filesystem::path &target);
SymlinkBlob(cpputils::unique_ref<blobstore::Blob> blob, std::function<void()> onDestruct);
SymlinkBlob(cpputils::unique_ref<blobstore::Blob> blob);
const boost::filesystem::path &target() const;

View File

@ -0,0 +1 @@
#include "DirBlobRef.h"

View File

@ -0,0 +1,80 @@
#ifndef CRYFS_FILESYSTEM_PARALLELACCESSFSBLOBSTORE_DIRBLOBREF_H
#define CRYFS_FILESYSTEM_PARALLELACCESSFSBLOBSTORE_DIRBLOBREF_H
#include "FsBlobRef.h"
#include "../fsblobstore/DirBlob.h"
namespace cryfs {
namespace parallelaccessfsblobstore {
class DirBlobRef: public FsBlobRef {
public:
DirBlobRef(fsblobstore::DirBlob *base): _base(base) {}
using Entry = fsblobstore::DirBlob::Entry;
const Entry &GetChild(const std::string &name) const {
return _base->GetChild(name);
}
const Entry &GetChild(const blockstore::Key &key) const {
return _base->GetChild(key);
}
void RemoveChild(const blockstore::Key &key) {
return _base->RemoveChild(key);
}
void flush() {
return _base->flush();
}
void AddChild(const std::string &name, const blockstore::Key &blobKey, fspp::Dir::EntryType type,
mode_t mode, uid_t uid, gid_t gid) {
return _base->AddChild(name, blobKey, type, mode, uid, gid);
}
void statChild(const blockstore::Key &key, struct ::stat *result) const {
return _base->statChild(key, result);
}
void chmodChild(const blockstore::Key &key, mode_t mode) {
return _base->chmodChild(key, mode);
}
void chownChild(const blockstore::Key &key, uid_t uid, gid_t gid) {
return _base->chownChild(key, uid, gid);
}
void AddChildDir(const std::string &name, const blockstore::Key &blobKey, mode_t mode, uid_t uid, gid_t gid) {
return _base->AddChildDir(name, blobKey, mode, uid, gid);
}
void AddChildFile(const std::string &name, const blockstore::Key &blobKey, mode_t mode, uid_t uid, gid_t gid) {
return _base->AddChildFile(name, blobKey, mode, uid, gid);
}
void AddChildSymlink(const std::string &name, const blockstore::Key &blobKey, uid_t uid, gid_t gid) {
return _base->AddChildSymlink(name, blobKey, uid, gid);
}
void AppendChildrenTo(std::vector<fspp::Dir::Entry> *result) const {
return _base->AppendChildrenTo(result);
}
const blockstore::Key &key() const {
return _base->key();
}
off_t lstat_size() const {
return _base->lstat_size();
}
private:
fsblobstore::DirBlob *_base;
};
}
}
#endif

View File

@ -0,0 +1 @@
#include "FileBlobRef.h"

View File

@ -0,0 +1,49 @@
#ifndef CRYFS_FILESYSTEM_PARALLELACCESSFSBLOBSTORE_FILEBLOBREF_H
#define CRYFS_FILESYSTEM_PARALLELACCESSFSBLOBSTORE_FILEBLOBREF_H
#include "FsBlobRef.h"
#include "../fsblobstore/FileBlob.h"
namespace cryfs {
namespace parallelaccessfsblobstore {
class FileBlobRef: public FsBlobRef {
public:
FileBlobRef(fsblobstore::FileBlob *base) : _base(base) {}
void resize(off_t size) {
return _base->resize(size);
}
off_t size() const {
return _base->size();
}
ssize_t read(void *target, uint64_t offset, uint64_t count) const {
return _base->read(target, offset, count);
}
void write(const void *source, uint64_t offset, uint64_t count) {
return _base->write(source, offset, count);
}
void flush() {
return _base->flush();
}
const blockstore::Key &key() const {
return _base->key();
}
off_t lstat_size() const {
return _base->lstat_size();
}
private:
fsblobstore::FileBlob *_base;
};
}
}
#endif

View File

@ -0,0 +1 @@
#include "FsBlobRef.h"

View File

@ -0,0 +1,25 @@
#ifndef CRYFS_PARALLELACCESSFSBLOBSTORE_FSBLOBREF_H
#define CRYFS_PARALLELACCESSFSBLOBSTORE_FSBLOBREF_H
#include <messmer/parallelaccessstore/ParallelAccessStore.h>
#include "../fsblobstore/FsBlob.h"
namespace cryfs {
namespace parallelaccessfsblobstore {
class FsBlobRef: public parallelaccessstore::ParallelAccessStore<fsblobstore::FsBlob, FsBlobRef, blockstore::Key>::ResourceRefBase {
public:
virtual const blockstore::Key &key() const = 0;
virtual off_t lstat_size() const = 0;
protected:
FsBlobRef() {}
private:
DISALLOW_COPY_AND_ASSIGN(FsBlobRef);
};
}
}
#endif

View File

@ -0,0 +1,89 @@
#include "ParallelAccessFsBlobStore.h"
#include "ParallelAccessFsBlobStoreAdapter.h"
#include "../fsblobstore/FsBlobStore.h"
namespace bf = boost::filesystem;
using cryfs::fsblobstore::FsBlobStore;
using cryfs::fsblobstore::FsBlob;
using cryfs::fsblobstore::FileBlob;
using cryfs::fsblobstore::DirBlob;
using cryfs::fsblobstore::SymlinkBlob;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
using blockstore::Key;
namespace cryfs {
namespace parallelaccessfsblobstore {
ParallelAccessFsBlobStore::ParallelAccessFsBlobStore(unique_ref<FsBlobStore> baseBlobStore)
: _baseBlobStore(std::move(baseBlobStore)),
_parallelAccessStore(make_unique_ref<ParallelAccessFsBlobStoreAdapter>(_baseBlobStore.get())) {
}
optional<unique_ref<FsBlobRef>> ParallelAccessFsBlobStore::load(const Key &key) {
return _parallelAccessStore.load(key, [this] (FsBlob *blob) {
FileBlob *fileBlob = dynamic_cast<FileBlob*>(blob);
if (fileBlob != nullptr) {
return unique_ref<FsBlobRef>(make_unique_ref<FileBlobRef>(fileBlob));
}
DirBlob *dirBlob = dynamic_cast<DirBlob*>(blob);
if (dirBlob != nullptr) {
dirBlob->setLstatSizeGetter(_getLstatSize());
return unique_ref<FsBlobRef>(make_unique_ref<DirBlobRef>(dirBlob));
}
SymlinkBlob *symlinkBlob = dynamic_cast<SymlinkBlob*>(blob);
if (symlinkBlob != nullptr) {
return unique_ref<FsBlobRef>(make_unique_ref<SymlinkBlobRef>(symlinkBlob));
}
ASSERT(false, "Unknown blob type loaded");
});
}
void ParallelAccessFsBlobStore::remove(unique_ref<FsBlobRef> blob) {
Key key = blob->key();
return _parallelAccessStore.remove(key, std::move(blob));
}
unique_ref<DirBlobRef> ParallelAccessFsBlobStore::createDirBlob() {
auto blob = _baseBlobStore->createDirBlob();
blob->setLstatSizeGetter(_getLstatSize());
Key key = blob->key();
return _parallelAccessStore.add<DirBlobRef>(key, std::move(blob), [] (FsBlob *resource) {
auto dirBlob = dynamic_cast<DirBlob*>(resource);
ASSERT(dirBlob != nullptr, "Wrong resource given");
return make_unique_ref<DirBlobRef>(dirBlob);
});
}
unique_ref<FileBlobRef> ParallelAccessFsBlobStore::createFileBlob() {
auto blob = _baseBlobStore->createFileBlob();
Key key = blob->key();
return _parallelAccessStore.add<FileBlobRef>(key, std::move(blob), [] (FsBlob *resource) {
auto fileBlob = dynamic_cast<FileBlob*>(resource);
ASSERT(fileBlob != nullptr, "Wrong resource given");
return make_unique_ref<FileBlobRef>(fileBlob);
});
}
unique_ref<SymlinkBlobRef> ParallelAccessFsBlobStore::createSymlinkBlob(const bf::path &target) {
auto blob = _baseBlobStore->createSymlinkBlob(target);
Key key = blob->key();
return _parallelAccessStore.add<SymlinkBlobRef>(key, std::move(blob), [] (FsBlob *resource) {
auto symlinkBlob = dynamic_cast<SymlinkBlob*>(resource);
ASSERT(symlinkBlob != nullptr, "Wrong resource given");
return make_unique_ref<SymlinkBlobRef>(symlinkBlob);
});
}
std::function<off_t (const blockstore::Key &key)> ParallelAccessFsBlobStore::_getLstatSize() {
return [this] (const blockstore::Key &key) {
auto blob = load(key);
ASSERT(blob != none, "Blob not found");
return (*blob)->lstat_size();
};
}
}
}

View File

@ -0,0 +1,36 @@
#ifndef CRYFS_FILESYSTEM_PARALLELACCESSFSBLOBSTORE_PARALLELACCESSFSBLOBSTORE_H
#define CRYFS_FILESYSTEM_PARALLELACCESSFSBLOBSTORE_PARALLELACCESSFSBLOBSTORE_H
#include <messmer/parallelaccessstore/ParallelAccessStore.h>
#include "FileBlobRef.h"
#include "DirBlobRef.h"
#include "SymlinkBlobRef.h"
#include "../fsblobstore/FsBlobStore.h"
namespace cryfs {
namespace parallelaccessfsblobstore {
//TODO Test classes in parallelaccessfsblobstore
class ParallelAccessFsBlobStore {
public:
ParallelAccessFsBlobStore(cpputils::unique_ref<fsblobstore::FsBlobStore> baseBlobStore);
cpputils::unique_ref<FileBlobRef> createFileBlob();
cpputils::unique_ref<DirBlobRef> createDirBlob();
cpputils::unique_ref<SymlinkBlobRef> createSymlinkBlob(const boost::filesystem::path &target);
boost::optional<cpputils::unique_ref<FsBlobRef>> load(const blockstore::Key &key);
void remove(cpputils::unique_ref<FsBlobRef> blob);
private:
cpputils::unique_ref<fsblobstore::FsBlobStore> _baseBlobStore;
parallelaccessstore::ParallelAccessStore<fsblobstore::FsBlob, FsBlobRef, blockstore::Key> _parallelAccessStore;
std::function<off_t (const blockstore::Key &)> _getLstatSize();
DISALLOW_COPY_AND_ASSIGN(ParallelAccessFsBlobStore);
};
}
}
#endif

View File

@ -0,0 +1 @@
#include "ParallelAccessFsBlobStoreAdapter.h"

View File

@ -0,0 +1,34 @@
#ifndef MESSMER_CRYFS_FILESYSTEM_PARALLELACCESSFSBLOBSTORE_PARALLELACCESSFSBLOBSTOREADAPTER_H_
#define MESSMER_CRYFS_FILESYSTEM_PARALLELACCESSFSBLOBSTORE_PARALLELACCESSFSBLOBSTOREADAPTER_H_
#include <messmer/cpp-utils/macros.h>
#include <messmer/parallelaccessstore/ParallelAccessStore.h>
#include "../fsblobstore/FsBlobStore.h"
namespace cryfs {
namespace parallelaccessfsblobstore {
class ParallelAccessFsBlobStoreAdapter: public parallelaccessstore::ParallelAccessBaseStore<fsblobstore::FsBlob, blockstore::Key> {
public:
explicit ParallelAccessFsBlobStoreAdapter(fsblobstore::FsBlobStore *baseBlockStore)
:_baseBlockStore(std::move(baseBlockStore)) {
}
boost::optional<cpputils::unique_ref<fsblobstore::FsBlob>> loadFromBaseStore(const blockstore::Key &key) override {
return _baseBlockStore->load(key);
}
void removeFromBaseStore(cpputils::unique_ref<fsblobstore::FsBlob> block) override {
return _baseBlockStore->remove(std::move(block));
}
private:
fsblobstore::FsBlobStore *_baseBlockStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessFsBlobStoreAdapter);
};
}
}
#endif

View File

@ -0,0 +1 @@
#include "SymlinkBlobRef.h"

View File

@ -0,0 +1,33 @@
#ifndef CRYFS_FILESYSTEM_PARALLELACCESSFSBLOBSTORE_SYMLINKBLOBREF_H
#define CRYFS_FILESYSTEM_PARALLELACCESSFSBLOBSTORE_SYMLINKBLOBREF_H
#include "FsBlobRef.h"
#include "../fsblobstore/SymlinkBlob.h"
namespace cryfs {
namespace parallelaccessfsblobstore {
class SymlinkBlobRef: public FsBlobRef {
public:
SymlinkBlobRef(fsblobstore::SymlinkBlob *base) : _base(base) {}
const boost::filesystem::path &target() const {
return _base->target();
}
const blockstore::Key &key() const {
return _base->key();
}
off_t lstat_size() const {
return _base->lstat_size();
}
private:
fsblobstore::SymlinkBlob *_base;
};
}
}
#endif

View File

@ -28,6 +28,7 @@ using std::vector;
//TODO Support files > 4GB
//TODO cryfs process doesn't seem to react to "kill". Needs "kill -9". Why? Furthermore, calling "fusermount -u" unmounts the fs, but the cryfs process keeps running. Why?
//TODO CryFS is only using 100% CPU (not parallel) when running bonnie. Furthermore, when calling "ls" in the mount/Bonnie.../ dir while bonnie runs, ls blocks and doesn't return. Probable reason: fsblobstore locks blobs instead of allowing parallel access. Use something like parallelaccessstore. Also generally improve parallelity.
void showVersion() {
cout << "CryFS Version " << version::VERSION_STRING << endl;