Report file system statistics (like free space) to operating system

src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp

@@ -28,29 +28,38 @@ using datatreestore::DataTreeStore;
 using parallelaccessdatatreestore::ParallelAccessDataTreeStore;
 
 BlobStoreOnBlocks::BlobStoreOnBlocks(unique_ref<BlockStore> blockStore, uint32_t blocksizeBytes)
-: _dataTreeStore(make_unique_ref<ParallelAccessDataTreeStore>(make_unique_ref<DataTreeStore>(make_unique_ref<DataNodeStore>(make_unique_ref<ParallelAccessBlockStore>(std::move(blockStore)), blocksizeBytes)))) {
+        : _dataTreeStore(make_unique_ref<ParallelAccessDataTreeStore>(make_unique_ref<DataTreeStore>(make_unique_ref<DataNodeStore>(make_unique_ref<ParallelAccessBlockStore>(std::move(blockStore)), blocksizeBytes)))) {
 }
 
 BlobStoreOnBlocks::~BlobStoreOnBlocks() {
 }
 
 unique_ref<Blob> BlobStoreOnBlocks::create() {
-  return make_unique_ref<BlobOnBlocks>(_dataTreeStore->createNewTree());
+    return make_unique_ref<BlobOnBlocks>(_dataTreeStore->createNewTree());
 }
 
 optional<unique_ref<Blob>> BlobStoreOnBlocks::load(const Key &key) {
-  auto tree = _dataTreeStore->load(key);
-  if (tree == none) {
-  	return none;
-  }
-  return optional<unique_ref<Blob>>(make_unique_ref<BlobOnBlocks>(std::move(*tree)));
+    auto tree = _dataTreeStore->load(key);
+    if (tree == none) {
+        return none;
+    }
+    return optional<unique_ref<Blob>>(make_unique_ref<BlobOnBlocks>(std::move(*tree)));
 }
 
 void BlobStoreOnBlocks::remove(unique_ref<Blob> blob) {
-  auto _blob = dynamic_pointer_move<BlobOnBlocks>(blob);
-  ASSERT(_blob != none, "Passed Blob in BlobStoreOnBlocks::remove() is not a BlobOnBlocks.");
-  _dataTreeStore->remove((*_blob)->releaseTree());
+    auto _blob = dynamic_pointer_move<BlobOnBlocks>(blob);
+    ASSERT(_blob != none, "Passed Blob in BlobStoreOnBlocks::remove() is not a BlobOnBlocks.");
+    _dataTreeStore->remove((*_blob)->releaseTree());
 }
 
+uint64_t BlobStoreOnBlocks::numBlocks() const {
+    return _dataTreeStore->numNodes();
+}
+
+uint64_t BlobStoreOnBlocks::estimateSpaceForNumBlocksLeft() const {
+    return _dataTreeStore->estimateSpaceForNumNodesLeft();
+}
+
+
 }
 }

src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h

@@ -3,6 +3,7 @@
 #define MESSMER_BLOBSTORE_IMPLEMENTATIONS_BLOCKED_BLOBSTOREONBLOCKS_H_
 
 #include "../../interface/BlobStore.h"
+#include "BlobOnBlocks.h"
 #include <blockstore/interface/BlockStore.h>
 
 namespace blobstore {
@@ -23,6 +24,10 @@ public:
 
   void remove(cpputils::unique_ref<Blob> blob) override;
 
+  //TODO Test numBlocks/estimateSpaceForNumBlocksLeft
+  uint64_t numBlocks() const override;
+  uint64_t estimateSpaceForNumBlocksLeft() const override;
+
 private:
   cpputils::unique_ref<parallelaccessdatatreestore::ParallelAccessDataTreeStore> _dataTreeStore;
 

src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp

@@ -92,6 +92,10 @@ uint64_t DataNodeStore::numNodes() const {
   return _blockstore->numBlocks();
 }
 
+uint64_t DataNodeStore::estimateSpaceForNumNodesLeft() const {
+  return _blockstore->estimateNumFreeBytes() / _layout.blocksizeBytes();
+}
+
 void DataNodeStore::removeSubtree(unique_ref<DataNode> node) {
   DataInnerNode *inner = dynamic_cast<DataInnerNode*>(node.get());
   if (inner != nullptr) {

src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h

@@ -41,7 +41,9 @@ public:
 
   void removeSubtree(cpputils::unique_ref<DataNode> node);
 
+  //TODO Test numBlocks/estimateSpaceForNumBlocksLeft
   uint64_t numNodes() const;
+  uint64_t estimateSpaceForNumNodesLeft() const;
   //TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree()
 
 private:

src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h

@@ -7,12 +7,10 @@
 #include <cpp-utils/pointer/unique_ref.h>
 #include <blockstore/utils/Key.h>
 #include <boost/optional.hpp>
+#include "../datanodestore/DataNodeStore.h"
 
 namespace blobstore {
 namespace onblocks {
-namespace datanodestore {
-class DataNodeStore;
-}
 namespace datatreestore {
 class DataTree;
 
@@ -27,12 +25,24 @@ public:
 
   void remove(cpputils::unique_ref<DataTree> tree);
 
+  //TODO Test numBlocks/estimateSpaceForNumBlocksLeft
+  uint64_t numNodes() const;
+  uint64_t estimateSpaceForNumNodesLeft() const;
+
 private:
   cpputils::unique_ref<datanodestore::DataNodeStore> _nodeStore;
 
   DISALLOW_COPY_AND_ASSIGN(DataTreeStore);
 };
 
+inline uint64_t DataTreeStore::numNodes() const {
+    return _nodeStore->numNodes();
+}
+
+inline uint64_t DataTreeStore::estimateSpaceForNumNodesLeft() const {
+    return _nodeStore->estimateSpaceForNumNodesLeft();
+}
+
 }
 }
 }

src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp

@@ -17,6 +17,8 @@ using datatreestore::DataTreeStore;
 using datatreestore::DataTree;
 namespace parallelaccessdatatreestore {
 
+//TODO Here and for other stores (DataTreeStore, ...): Make small functions inline
+
 ParallelAccessDataTreeStore::ParallelAccessDataTreeStore(unique_ref<DataTreeStore> dataTreeStore)
   : _dataTreeStore(std::move(dataTreeStore)), _parallelAccessStore(make_unique_ref<ParallelAccessDataTreeStoreAdapter>(_dataTreeStore.get())) {
 }
@@ -39,6 +41,8 @@ void ParallelAccessDataTreeStore::remove(unique_ref<DataTreeRef> tree) {
   return _parallelAccessStore.remove(key, std::move(tree));
 }
 
+
+
 }
 }
 }

src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h

@@ -6,13 +6,10 @@
 #include <cpp-utils/macros.h>
 #include <blockstore/utils/Key.h>
 #include <parallelaccessstore/ParallelAccessStore.h>
+#include "../datatreestore/DataTreeStore.h"
 
 namespace blobstore {
 namespace onblocks {
-namespace datatreestore {
-class DataTreeStore;
-class DataTree;
-}
 namespace parallelaccessdatatreestore {
 class DataTreeRef;
 
@@ -29,6 +26,10 @@ public:
 
   void remove(cpputils::unique_ref<DataTreeRef> tree);
 
+  //TODO Test numBlocks/estimateSpaceForNumBlocksLeft
+  uint64_t numNodes() const;
+  uint64_t estimateSpaceForNumNodesLeft() const;
+
 private:
   cpputils::unique_ref<datatreestore::DataTreeStore> _dataTreeStore;
   parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::Key> _parallelAccessStore;
@@ -36,6 +37,14 @@ private:
   DISALLOW_COPY_AND_ASSIGN(ParallelAccessDataTreeStore);
 };
 
+inline uint64_t ParallelAccessDataTreeStore::numNodes() const {
+    return _dataTreeStore->numNodes();
+}
+
+inline uint64_t ParallelAccessDataTreeStore::estimateSpaceForNumNodesLeft() const {
+    return _dataTreeStore->estimateSpaceForNumNodesLeft();
+}
+
 }
 }
 }

src/blobstore/interface/BlobStore.h

@@ -18,6 +18,9 @@ public:
   virtual cpputils::unique_ref<Blob> create() = 0;
   virtual boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::Key &key) = 0;
   virtual void remove(cpputils::unique_ref<Blob> blob) = 0;
+
+  virtual uint64_t numBlocks() const = 0;
+  virtual uint64_t estimateSpaceForNumBlocksLeft() const = 0;
 };
 
 }

src/blockstore/implementations/caching/CachingBlockStore.cpp

@@ -67,6 +67,10 @@ uint64_t CachingBlockStore::numBlocks() const {
   return _baseBlockStore->numBlocks() + _numNewBlocks;
 }
 
+uint64_t CachingBlockStore::estimateNumFreeBytes() const {
+  return _baseBlockStore->estimateNumFreeBytes();
+}
+
 void CachingBlockStore::release(unique_ref<Block> block) {
   Key key = block->key();
   _cache.push(key, std::move(block));

src/blockstore/implementations/caching/CachingBlockStore.h

@@ -18,6 +18,7 @@ public:
   boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
 
   void release(cpputils::unique_ref<Block> block);
 

src/blockstore/implementations/compressing/CompressingBlockStore.h

@@ -19,6 +19,7 @@ public:
     boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
     void remove(cpputils::unique_ref<Block> block) override;
     uint64_t numBlocks() const override;
+    uint64_t estimateNumFreeBytes() const override;
 
 private:
     cpputils::unique_ref<BlockStore> _baseBlockStore;
@@ -71,6 +72,11 @@ uint64_t CompressingBlockStore<Compressor>::numBlocks() const {
     return _baseBlockStore->numBlocks();
 }
 
+template<class Compressor>
+uint64_t CompressingBlockStore<Compressor>::estimateNumFreeBytes() const {
+    return _baseBlockStore->estimateNumFreeBytes();
+}
+
 }
 }
 

src/blockstore/implementations/encrypted/EncryptedBlockStore.h

@@ -22,6 +22,7 @@ public:
   boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
 
   //This function should only be used by test cases
   void __setKey(const typename Cipher::EncryptionKey &encKey);
@@ -79,6 +80,11 @@ uint64_t EncryptedBlockStore<Cipher>::numBlocks() const {
   return _baseBlockStore->numBlocks();
 }
 
+template<class Cipher>
+uint64_t EncryptedBlockStore<Cipher>::estimateNumFreeBytes() const {
+  return _baseBlockStore->estimateNumFreeBytes();
+}
+
 template<class Cipher>
 void EncryptedBlockStore<Cipher>::__setKey(const typename Cipher::EncryptionKey &encKey) {
   _encKey = encKey;

src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp

@@ -52,5 +52,12 @@ uint64_t InMemoryBlockStore::numBlocks() const {
   return _blocks.size();
 }
 
+uint64_t InMemoryBlockStore::estimateNumFreeBytes() const {
+  //For windows, see http://stackoverflow.com/a/2513561/829568
+  long numRAMPages = sysconf(_SC_PHYS_PAGES);
+  long pageSize = sysconf(_SC_PAGE_SIZE);
+  return numRAMPages*pageSize;
+}
+
 }
 }

src/blockstore/implementations/inmemory/InMemoryBlockStore.h

@@ -20,6 +20,7 @@ public:
   boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
 
 private:
   std::map<std::string, InMemoryBlock> _blocks;

src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp

@@ -1,5 +1,6 @@
 #include "OnDiskBlock.h"
 #include "OnDiskBlockStore.h"
+#include <sys/statvfs.h>
 
 using std::string;
 using cpputils::Data;
@@ -48,5 +49,11 @@ uint64_t OnDiskBlockStore::numBlocks() const {
   return std::distance(bf::directory_iterator(_rootdir), bf::directory_iterator());
 }
 
+uint64_t OnDiskBlockStore::estimateNumFreeBytes() const {
+  struct statvfs stat;
+  ::statvfs(_rootdir.c_str(), &stat);
+  return stat.f_bsize*stat.f_bavail;
+}
+
 }
 }

src/blockstore/implementations/ondisk/OnDiskBlockStore.h

@@ -18,6 +18,7 @@ public:
   boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
 
 private:
   const boost::filesystem::path _rootdir;

src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.cpp

@@ -56,5 +56,9 @@ uint64_t ParallelAccessBlockStore::numBlocks() const {
   return _baseBlockStore->numBlocks();
 }
 
+uint64_t ParallelAccessBlockStore::estimateNumFreeBytes() const {
+  return _baseBlockStore->estimateNumFreeBytes();
+}
+
 }
 }

src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h

@@ -20,6 +20,7 @@ public:
   boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
 
 private:
   cpputils::unique_ref<BlockStore> _baseBlockStore;

src/blockstore/implementations/testfake/FakeBlockStore.cpp

@@ -76,5 +76,12 @@ uint64_t FakeBlockStore::numBlocks() const {
   return _blocks.size();
 }
 
+uint64_t FakeBlockStore::estimateNumFreeBytes() const {
+  //For windows, see http://stackoverflow.com/a/2513561/829568
+  long numRAMPages = sysconf(_SC_PHYS_PAGES);
+  long pageSize = sysconf(_SC_PAGE_SIZE);
+  return numRAMPages*pageSize;
+}
+
 }
 }

src/blockstore/implementations/testfake/FakeBlockStore.h

@@ -35,6 +35,7 @@ public:
   boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
   void remove(cpputils::unique_ref<Block> block) override;
   uint64_t numBlocks() const override;
+  uint64_t estimateNumFreeBytes() const override;
 
   void updateData(const Key &key, const cpputils::Data &data);
 

src/blockstore/interface/BlockStore.h

@@ -22,6 +22,8 @@ public:
   virtual boost::optional<cpputils::unique_ref<Block>> load(const Key &key) = 0;
   virtual void remove(cpputils::unique_ref<Block> block) = 0;
   virtual uint64_t numBlocks() const = 0;
+  //TODO Test estimateNumFreeBytes in all block stores
+  virtual uint64_t estimateNumFreeBytes() const = 0;
 
   cpputils::unique_ref<Block> create(const cpputils::Data &data) {
     while(true) {

src/cryfs/filesystem/CryDevice.cpp

@@ -138,7 +138,16 @@ unique_ref<FsBlobRef> CryDevice::LoadBlob(const bf::path &path) {
 
 void CryDevice::statfs(const bf::path &path, struct statvfs *fsstat) {
   callFsActionCallbacks();
-  // TODO What should we report here?
+  uint64_t numUsedBlocks = _fsBlobStore->numBlocks();
+  uint64_t numFreeBlocks = _fsBlobStore->estimateSpaceForNumBlocksLeft();
+  fsstat->f_bsize = BLOCKSIZE_BYTES;
+  fsstat->f_blocks = numUsedBlocks + numFreeBlocks;
+  fsstat->f_bfree = numFreeBlocks;
+  fsstat->f_bavail = numFreeBlocks;
+  fsstat->f_files = numUsedBlocks + numFreeBlocks;
+  fsstat->f_ffree = numFreeBlocks;
+  fsstat->f_namemax = 255; // We theoretically support unlimited file name length, but this is default for many Linux file systems, so probably also makes sense for CryFS.
+  //f_frsize, f_favail, f_fsid and f_flag are ignored in fuse, see http://fuse.sourcearchive.com/documentation/2.7.0/structfuse__operations_4e765e29122e7b6b533dc99849a52655.html#4e765e29122e7b6b533dc99849a52655
 }
 
 unique_ref<FileBlobRef> CryDevice::CreateFileBlob() {

src/cryfs/filesystem/cachingfsblobstore/CachingFsBlobStore.h

@@ -24,6 +24,8 @@ namespace cryfs {
             cpputils::unique_ref<SymlinkBlobRef> createSymlinkBlob(const boost::filesystem::path &target);
             boost::optional<cpputils::unique_ref<FsBlobRef>> load(const blockstore::Key &key);
             void remove(cpputils::unique_ref<FsBlobRef> blob);
+            uint64_t numBlocks() const;
+            uint64_t estimateSpaceForNumBlocksLeft() const;
 
             void releaseForCache(cpputils::unique_ref<fsblobstore::FsBlob> baseBlob);
 
@@ -78,6 +80,14 @@ namespace cryfs {
             _cache.push(key, std::move(baseBlob));
         }
 
+        inline uint64_t CachingFsBlobStore::numBlocks() const {
+            return _baseBlobStore->numBlocks();
+        }
+
+        inline uint64_t CachingFsBlobStore::estimateSpaceForNumBlocksLeft() const {
+            return _baseBlobStore->estimateSpaceForNumBlocksLeft();
+        }
+
     }
 }
 

src/cryfs/filesystem/fsblobstore/FsBlobStore.h

@@ -22,6 +22,8 @@ namespace cryfs {
             cpputils::unique_ref<SymlinkBlob> createSymlinkBlob(const boost::filesystem::path &target);
             boost::optional<cpputils::unique_ref<FsBlob>> load(const blockstore::Key &key);
             void remove(cpputils::unique_ref<FsBlob> blob);
+            uint64_t numBlocks() const;
+            uint64_t estimateSpaceForNumBlocksLeft() const;
 
         private:
 
@@ -51,6 +53,14 @@ namespace cryfs {
             return SymlinkBlob::InitializeSymlink(std::move(blob), target);
         }
 
+        inline uint64_t FsBlobStore::numBlocks() const {
+            return _baseBlobStore->numBlocks();
+        }
+
+        inline uint64_t FsBlobStore::estimateSpaceForNumBlocksLeft() const {
+            return _baseBlobStore->estimateSpaceForNumBlocksLeft();
+        }
+
         inline void FsBlobStore::remove(cpputils::unique_ref<FsBlob> blob) {
             _baseBlobStore->remove(blob->releaseBaseBlob());
         }

src/cryfs/filesystem/parallelaccessfsblobstore/ParallelAccessFsBlobStore.h

@@ -26,6 +26,8 @@ namespace cryfs {
             cpputils::unique_ref<SymlinkBlobRef> createSymlinkBlob(const boost::filesystem::path &target);
             boost::optional<cpputils::unique_ref<FsBlobRef>> load(const blockstore::Key &key);
             void remove(cpputils::unique_ref<FsBlobRef> blob);
+            uint64_t numBlocks() const;
+            uint64_t estimateSpaceForNumBlocksLeft() const;
 
         private:
 
@@ -42,19 +44,26 @@ namespace cryfs {
                   _parallelAccessStore(cpputils::make_unique_ref<ParallelAccessFsBlobStoreAdapter>(_baseBlobStore.get())) {
         }
 
-        void ParallelAccessFsBlobStore::remove(cpputils::unique_ref<FsBlobRef> blob) {
+        inline void ParallelAccessFsBlobStore::remove(cpputils::unique_ref<FsBlobRef> blob) {
             blockstore::Key key = blob->key();
             return _parallelAccessStore.remove(key, std::move(blob));
         }
 
-        std::function<off_t (const blockstore::Key &key)> ParallelAccessFsBlobStore::_getLstatSize() {
+        inline std::function<off_t (const blockstore::Key &key)> ParallelAccessFsBlobStore::_getLstatSize() {
             return [this] (const blockstore::Key &key) {
                 auto blob = load(key);
                 ASSERT(blob != boost::none, "Blob not found");
                 return (*blob)->lstat_size();
             };
         }
-        
+
+        inline uint64_t ParallelAccessFsBlobStore::numBlocks() const {
+            return _baseBlobStore->numBlocks();
+        }
+
+        inline uint64_t ParallelAccessFsBlobStore::estimateSpaceForNumBlocksLeft() const {
+            return _baseBlobStore->estimateSpaceForNumBlocksLeft();
+        }
     }
 }