From ab3b5d906c4523207517434ee9401df6e3f89bbc Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Fri, 5 Dec 2014 06:20:38 +0100 Subject: [PATCH] Written a ondisk blobstore implementation and started with test cases --- src/blobstore/CMakeLists.txt | 1 + src/blobstore/implementations/CMakeLists.txt | 1 + .../implementations/ondisk/CMakeLists.txt | 3 + src/blobstore/implementations/ondisk/Data.cpp | 29 +++++++++ src/blobstore/implementations/ondisk/Data.h | 30 +++++++++ .../implementations/ondisk/OnDiskBlob.cpp | 45 +++++++++++++ .../implementations/ondisk/OnDiskBlob.h | 36 +++++++++++ .../ondisk/OnDiskBlobStore.cpp | 63 +++++++++++++++++++ .../implementations/ondisk/OnDiskBlobStore.h | 32 ++++++++++ src/test/blobstore/ondisk/DataTest.cpp | 63 +++++++++++++++++++ 10 files changed, 303 insertions(+) create mode 100644 src/blobstore/CMakeLists.txt create mode 100644 src/blobstore/implementations/CMakeLists.txt create mode 100644 src/blobstore/implementations/ondisk/CMakeLists.txt create mode 100644 src/blobstore/implementations/ondisk/Data.cpp create mode 100644 src/blobstore/implementations/ondisk/Data.h create mode 100644 src/blobstore/implementations/ondisk/OnDiskBlob.cpp create mode 100644 src/blobstore/implementations/ondisk/OnDiskBlob.h create mode 100644 src/blobstore/implementations/ondisk/OnDiskBlobStore.cpp create mode 100644 src/blobstore/implementations/ondisk/OnDiskBlobStore.h create mode 100644 src/test/blobstore/ondisk/DataTest.cpp diff --git a/src/blobstore/CMakeLists.txt b/src/blobstore/CMakeLists.txt new file mode 100644 index 00000000..f21d8c7e --- /dev/null +++ b/src/blobstore/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(implementations) diff --git a/src/blobstore/implementations/CMakeLists.txt b/src/blobstore/implementations/CMakeLists.txt new file mode 100644 index 00000000..8debea1a --- /dev/null +++ b/src/blobstore/implementations/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(ondisk) diff --git a/src/blobstore/implementations/ondisk/CMakeLists.txt b/src/blobstore/implementations/ondisk/CMakeLists.txt new file mode 100644 index 00000000..02e1d800 --- /dev/null +++ b/src/blobstore/implementations/ondisk/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(blobstore_ondisk Data.cpp OnDiskBlob.cpp OnDiskBlobStore.cpp) + +target_link_libraries(blobstore_ondisk boost_filesystem boost_system) diff --git a/src/blobstore/implementations/ondisk/Data.cpp b/src/blobstore/implementations/ondisk/Data.cpp new file mode 100644 index 00000000..29bcbfa0 --- /dev/null +++ b/src/blobstore/implementations/ondisk/Data.cpp @@ -0,0 +1,29 @@ +#include + +#include + +namespace blobstore { +namespace ondisk { + +Data::Data(size_t size) +: _data(std::malloc(size)) { + if (nullptr == _data) { + throw std::bad_alloc(); + } +} + +Data::~Data() { + std::free(_data); + _data = nullptr; +} + +void *Data::data() { + return const_cast(const_cast(this)->data()); +} + +const void *Data::data() const { + return _data; +} + +} /* namespace ondisk */ +} /* namespace blobstore */ diff --git a/src/blobstore/implementations/ondisk/Data.h b/src/blobstore/implementations/ondisk/Data.h new file mode 100644 index 00000000..8aafd3b3 --- /dev/null +++ b/src/blobstore/implementations/ondisk/Data.h @@ -0,0 +1,30 @@ +#pragma once +#ifndef BLOBSTORE_IMPLEMENTATIONS_ONDISK_DATA_H_ +#define BLOBSTORE_IMPLEMENTATIONS_ONDISK_DATA_H_ + +#include +//TODO Move this to a more generic utils +#include "fspp/utils/macros.h" + +namespace blobstore { +namespace ondisk { + +class Data { +public: + Data(size_t size); + virtual ~Data(); + + void *data(); + const void *data() const; + +private: + void *_data; + + DISALLOW_COPY_AND_ASSIGN(Data); +}; + + +} /* namespace ondisk */ +} /* namespace blobstore */ + +#endif diff --git a/src/blobstore/implementations/ondisk/OnDiskBlob.cpp b/src/blobstore/implementations/ondisk/OnDiskBlob.cpp new file mode 100644 index 00000000..48f8104e --- /dev/null +++ b/src/blobstore/implementations/ondisk/OnDiskBlob.cpp @@ -0,0 +1,45 @@ +#include "OnDiskBlob.h" + +#include "OnDiskBlobStore.h" + +#include + +using std::istream; +using std::ostream; + +namespace blobstore { +namespace ondisk { + +OnDiskBlob::OnDiskBlob(size_t size) + : _size(size), _data(size) { +} + +OnDiskBlob::~OnDiskBlob() { +} + +void *OnDiskBlob::data() { + return _data.data(); +} + +const void *OnDiskBlob::data() const { + return _data.data(); +} + +size_t OnDiskBlob::size() const { + return _size; +} + +void OnDiskBlob::LoadDataFromStream(istream &stream) { + stream.read((char*)_data.data(), _size); +} + +void OnDiskBlob::StoreDataToStream(ostream &stream) const { + stream.write((const char*)_data.data(), _size); +} + +void OnDiskBlob::FillDataWithZeroes() { + std::memset(_data.data(), 0, _size); +} + +} /* namespace ondisk */ +} /* namespace blobstore */ diff --git a/src/blobstore/implementations/ondisk/OnDiskBlob.h b/src/blobstore/implementations/ondisk/OnDiskBlob.h new file mode 100644 index 00000000..d6975504 --- /dev/null +++ b/src/blobstore/implementations/ondisk/OnDiskBlob.h @@ -0,0 +1,36 @@ +#pragma once +#ifndef BLOBSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOB_H_ +#define BLOBSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOB_H_ + +#include "blobstore/interface/Blob.h" +#include "Data.h" + +#include + +namespace blobstore { +namespace ondisk { +class OnDiskBlobStore; + +class OnDiskBlob: public Blob { +public: + OnDiskBlob(size_t size); + virtual ~OnDiskBlob(); + + void *data() override; + const void *data() const override; + + size_t size() const override; + + void LoadDataFromStream(std::istream &stream); + void StoreDataToStream(std::ostream &stream) const; + void FillDataWithZeroes(); + +private: + size_t _size; + Data _data; +}; + +} /* namespace ondisk */ +} /* namespace blobstore */ + +#endif diff --git a/src/blobstore/implementations/ondisk/OnDiskBlobStore.cpp b/src/blobstore/implementations/ondisk/OnDiskBlobStore.cpp new file mode 100644 index 00000000..cfd28f79 --- /dev/null +++ b/src/blobstore/implementations/ondisk/OnDiskBlobStore.cpp @@ -0,0 +1,63 @@ +#include "OnDiskBlobStore.h" + +#include "OnDiskBlob.h" + +#include + +using std::unique_ptr; +using std::make_unique; +using std::istream; +using std::ifstream; +using std::ofstream; +using std::ios; + +namespace blobstore { +namespace ondisk { + +OnDiskBlobStore::OnDiskBlobStore(const boost::filesystem::path &rootdir) + : _rootdir(rootdir) {} + +unique_ptr OnDiskBlobStore::create(const std::string &key, size_t size) { + auto blob = make_unique(size); + blob->FillDataWithZeroes(); + _storeBlobData(key, blob.get()); + return std::move(blob); +} + +void OnDiskBlobStore::_storeBlobData(const std::string &key, const OnDiskBlob *blob) { + auto file_path = _rootdir / key; + ofstream file(file_path.c_str(), ios::binary | ios::trunc); + + blob->StoreDataToStream(file); +} + +unique_ptr OnDiskBlobStore::load(const std::string &key) { + auto file_path = _rootdir / key; + ifstream file(file_path.c_str(), ios::binary); + + return _createBlobFromStream(file); +} + +unique_ptr OnDiskBlobStore::_createBlobFromStream(istream &stream) { + size_t size = _getStreamSize(stream); + + auto blob = make_unique(size); + blob->LoadDataFromStream(stream); + return std::move(blob); +} + +size_t OnDiskBlobStore::_getStreamSize(istream &stream) { + auto current_pos = stream.tellg(); + + //Retrieve length + stream.seekg(0, stream.end); + auto endpos = stream.tellg(); + + //Restore old position + stream.seekg(current_pos, stream.beg); + + return endpos - current_pos; +} + +} /* namespace ondisk */ +} /* namespace blobstore */ diff --git a/src/blobstore/implementations/ondisk/OnDiskBlobStore.h b/src/blobstore/implementations/ondisk/OnDiskBlobStore.h new file mode 100644 index 00000000..3b60e189 --- /dev/null +++ b/src/blobstore/implementations/ondisk/OnDiskBlobStore.h @@ -0,0 +1,32 @@ +#pragma once +#ifndef BLOBSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOBSTORE_H_ +#define BLOBSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOBSTORE_H_ + +#include "blobstore/interface/BlobStore.h" + +#include +#include + +namespace blobstore { +namespace ondisk { +class OnDiskBlob; + +class OnDiskBlobStore: public BlobStore { +public: + OnDiskBlobStore(const boost::filesystem::path &rootdir); + + std::unique_ptr create(const std::string &key, size_t size) override; + std::unique_ptr load(const std::string &key) override; + +private: + const boost::filesystem::path _rootdir; + + void _storeBlobData(const std::string &key, const OnDiskBlob *blob); + std::unique_ptr _createBlobFromStream(std::istream &stream); + size_t _getStreamSize(std::istream &stream); +}; + +} /* namespace ondisk */ +} /* namespace blobstore */ + +#endif diff --git a/src/test/blobstore/ondisk/DataTest.cpp b/src/test/blobstore/ondisk/DataTest.cpp new file mode 100644 index 00000000..5e93c12f --- /dev/null +++ b/src/test/blobstore/ondisk/DataTest.cpp @@ -0,0 +1,63 @@ +#include "gtest/gtest.h" + +#include "blobstore/implementations/ondisk/Data.h" + +using ::testing::Test; + +using blobstore::ondisk::Data; + +class DataTest: public Test { +public: + void FillData(char *data, size_t size) { + for (size_t i = 0; i < size; ++i) { + data[i] = i; + } + } + + void CheckData(char *data, size_t size) { + for (size_t i = 0; i < size; ++i) { + EXPECT_EQ((char)i, data[i]); + } + } +}; + +TEST_F(DataTest, EmptyData) { + Data data(0); +} + +TEST_F(DataTest, OneByteData) { + Data data(1); + ((char*)data.data())[0] = 0x3E; + + EXPECT_EQ(0x3E, ((char*)data.data())[0]); +} + +TEST_F(DataTest, MidsizeData) { + Data data(4096); + + FillData((char*)data.data(), 4096); + CheckData((char*)data.data(), 4096); +} + +// Working on a large data area without a crash is a good indicator that we +// are actually working on memory that was validly allocated for us. +TEST_F(DataTest, LargeData) { + Data data(10 * 1024 * 1024); + + FillData((char*)data.data(), 10 * 1024 * 1024); + CheckData((char*)data.data(), 10 * 1024 * 1024); +} + +// This test doesn't ensure that the Data class gives the memory region free, +// but it is a good indicator. +TEST_F(DataTest, InaccessibleAfterDeletion) { + Data *data = new Data(1); + ((char*)data->data())[0] = 0x3E; // Access data byte 0 + + delete data; + + EXPECT_DEATH( + ((char*)data->data())[0] = 0x3E, + "" + ); +}