2016-03-01 17:45:48 +01:00
# include <iostream>
# include <boost/filesystem.hpp>
2019-01-26 08:38:34 +01:00
# include <cryfs/impl/config/CryConfigLoader.h>
# include <cryfs/impl/config/CryPasswordBasedKeyProvider.h>
2017-07-18 23:49:51 +02:00
# include <blockstore/implementations/ondisk/OnDiskBlockStore2.h>
2018-11-10 21:38:57 +01:00
# include <blockstore/implementations/integrity/IntegrityBlockStore2.h>
2017-07-18 23:49:51 +02:00
# include <blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h>
2016-03-01 17:45:48 +01:00
# include <blobstore/implementations/onblocks/datanodestore/DataNodeStore.h>
# include <blobstore/implementations/onblocks/datanodestore/DataNode.h>
# include <blobstore/implementations/onblocks/datanodestore/DataInnerNode.h>
# include <blobstore/implementations/onblocks/datanodestore/DataLeafNode.h>
# include <blobstore/implementations/onblocks/BlobStoreOnBlocks.h>
2019-01-26 08:38:34 +01:00
# include <cryfs/impl/filesystem/fsblobstore/FsBlobStore.h>
# include <cryfs/impl/filesystem/fsblobstore/DirBlob.h>
# include <cryfs/impl/filesystem/CryDevice.h>
2018-10-22 04:31:08 +02:00
# include <cpp-utils/io/IOStreamConsole.h>
2018-11-10 21:38:57 +01:00
# include <cpp-utils/system/homedir.h>
2019-05-26 11:34:26 +02:00
# include "traversal.h"
2016-03-01 17:45:48 +01:00
2018-07-30 08:28:35 +02:00
# include <set>
2019-06-02 06:05:26 +02:00
using std : : endl ;
using std : : cout ;
using std : : set ;
using std : : flush ;
using std : : vector ;
using boost : : none ;
using boost : : filesystem : : path ;
2016-03-01 17:45:48 +01:00
using namespace cryfs ;
using namespace cpputils ;
using namespace blockstore ;
using namespace blockstore : : ondisk ;
2018-11-10 21:38:57 +01:00
using namespace blockstore : : integrity ;
2017-07-18 23:49:51 +02:00
using namespace blockstore : : lowtohighlevel ;
2016-03-01 17:45:48 +01:00
using namespace blobstore : : onblocks ;
using namespace blobstore : : onblocks : : datanodestore ;
using namespace cryfs : : fsblobstore ;
2019-05-26 11:34:26 +02:00
using namespace cryfs_stats ;
2016-03-01 17:45:48 +01:00
void printNode ( unique_ref < DataNode > node ) {
2018-11-10 21:38:57 +01:00
std : : cout < < " BlockId: " < < node - > blockId ( ) . ToString ( ) < < " , Depth: " < < static_cast < int > ( node - > depth ( ) ) < < " " ;
2016-03-01 17:45:48 +01:00
auto innerNode = dynamic_pointer_move < DataInnerNode > ( node ) ;
if ( innerNode ! = none ) {
std : : cout < < " Type: inner \n " ;
return ;
}
auto leafNode = dynamic_pointer_move < DataLeafNode > ( node ) ;
if ( leafNode ! = none ) {
std : : cout < < " Type: leaf \n " ;
return ;
}
}
2018-11-10 21:38:57 +01:00
unique_ref < BlockStore > makeBlockStore ( const path & basedir , const CryConfigLoader : : ConfigLoadResult & config , LocalStateDir & localStateDir ) {
auto onDiskBlockStore = make_unique_ref < OnDiskBlockStore2 > ( basedir ) ;
2018-12-22 00:58:30 +01:00
auto encryptedBlockStore = CryCiphers : : find ( config . configFile - > config ( ) - > Cipher ( ) ) . createEncryptedBlockstore ( std : : move ( onDiskBlockStore ) , config . configFile - > config ( ) - > EncryptionKey ( ) ) ;
auto statePath = localStateDir . forFilesystemId ( config . configFile - > config ( ) - > FilesystemId ( ) ) ;
2018-11-10 21:38:57 +01:00
auto integrityFilePath = statePath / " integritydata " ;
2018-12-11 06:20:18 +01:00
auto onIntegrityViolation = [ ] ( ) {
std : : cerr < < " Warning: Integrity violation encountered " < < std : : endl ;
} ;
auto integrityBlockStore = make_unique_ref < IntegrityBlockStore2 > ( std : : move ( encryptedBlockStore ) , integrityFilePath , config . myClientId , false , true , onIntegrityViolation ) ;
2018-11-10 21:38:57 +01:00
return make_unique_ref < LowToHighLevelBlockStore > ( std : : move ( integrityBlockStore ) ) ;
}
2019-05-26 11:34:26 +02:00
struct AccumulateBlockIds final {
public :
auto callback ( ) {
return [ this ] ( const BlockId & id ) {
_blockIds . push_back ( id ) ;
} ;
}
const std : : vector < BlockId > & blockIds ( ) const {
return _blockIds ;
}
void reserve ( size_t size ) {
_blockIds . reserve ( size ) ;
}
private :
std : : vector < BlockId > _blockIds ;
} ;
class ProgressBar final {
public :
ProgressBar ( size_t numBlocks ) : _currentBlock ( 0 ) , _numBlocks ( numBlocks ) { }
auto callback ( ) {
return [ this ] ( const BlockId & ) {
cout < < " \r " < < ( + + _currentBlock ) < < " / " < < _numBlocks < < flush ;
} ;
}
private :
size_t _currentBlock ;
size_t _numBlocks ;
} ;
2018-11-10 21:38:57 +01:00
std : : vector < BlockId > _getKnownBlobIds ( const path & basedir , const CryConfigLoader : : ConfigLoadResult & config , LocalStateDir & localStateDir ) {
auto blockStore = makeBlockStore ( basedir , config , localStateDir ) ;
2018-12-22 00:58:30 +01:00
auto fsBlobStore = make_unique_ref < FsBlobStore > ( make_unique_ref < BlobStoreOnBlocks > ( std : : move ( blockStore ) , config . configFile - > config ( ) - > BlocksizeBytes ( ) ) ) ;
2018-11-10 21:38:57 +01:00
std : : vector < BlockId > result ;
2019-05-26 11:34:26 +02:00
AccumulateBlockIds knownBlobIds ;
2018-11-10 21:38:57 +01:00
cout < < " Listing all file system entities (i.e. blobs)... " < < flush ;
2018-12-22 00:58:30 +01:00
auto rootId = BlockId : : FromString ( config . configFile - > config ( ) - > RootBlob ( ) ) ;
2019-05-26 11:34:26 +02:00
forEachReachableBlob ( fsBlobStore . get ( ) , rootId , { knownBlobIds . callback ( ) } ) ;
2018-11-10 21:38:57 +01:00
cout < < " done " < < endl ;
2019-05-26 11:34:26 +02:00
return knownBlobIds . blockIds ( ) ;
2018-11-10 21:38:57 +01:00
}
std : : vector < BlockId > _getKnownBlockIds ( const path & basedir , const CryConfigLoader : : ConfigLoadResult & config , LocalStateDir & localStateDir ) {
auto knownBlobIds = _getKnownBlobIds ( basedir , config , localStateDir ) ;
auto blockStore = makeBlockStore ( basedir , config , localStateDir ) ;
2018-12-22 00:58:30 +01:00
auto nodeStore = make_unique_ref < DataNodeStore > ( std : : move ( blockStore ) , config . configFile - > config ( ) - > BlocksizeBytes ( ) ) ;
2019-05-26 11:34:26 +02:00
AccumulateBlockIds knownBlockIds ;
2018-11-10 21:38:57 +01:00
const uint32_t numNodes = nodeStore - > numNodes ( ) ;
2019-05-26 11:34:26 +02:00
knownBlockIds . reserve ( numNodes ) ;
2018-11-10 21:38:57 +01:00
cout < < " Listing all blocks used by these file system entities... " < < endl ;
for ( const auto & blobId : knownBlobIds ) {
2019-05-26 11:34:26 +02:00
forEachReachableBlockInBlob ( nodeStore . get ( ) , blobId , {
ProgressBar ( numNodes ) . callback ( ) ,
knownBlockIds . callback ( )
2018-11-10 21:38:57 +01:00
} ) ;
2016-03-01 17:45:48 +01:00
}
2018-11-10 21:38:57 +01:00
std : : cout < < " ...done " < < endl ;
2019-05-26 11:34:26 +02:00
return knownBlockIds . blockIds ( ) ;
2016-03-01 17:45:48 +01:00
}
2018-11-10 21:38:57 +01:00
set < BlockId > _getAllBlockIds ( const path & basedir , const CryConfigLoader : : ConfigLoadResult & config , LocalStateDir & localStateDir ) {
2019-05-26 11:34:26 +02:00
auto blockStore = makeBlockStore ( basedir , config , localStateDir ) ;
AccumulateBlockIds allBlockIds ;
allBlockIds . reserve ( blockStore - > numBlocks ( ) ) ;
forEachBlock ( blockStore . get ( ) , { allBlockIds . callback ( ) } ) ;
return set < BlockId > ( allBlockIds . blockIds ( ) . begin ( ) , allBlockIds . blockIds ( ) . end ( ) ) ;
2018-11-10 21:38:57 +01:00
}
int main ( int argc , char * argv [ ] ) {
if ( argc ! = 2 ) {
std : : cerr < < " Usage: cryfs-stats [basedir] " < < std : : endl ;
exit ( 1 ) ;
}
path basedir = argv [ 1 ] ;
std : : cout < < " Calculating stats for filesystem at " < < basedir < < std : : endl ;
2016-03-01 17:45:48 +01:00
2018-10-22 04:31:08 +02:00
auto console = std : : make_shared < cpputils : : IOStreamConsole > ( ) ;
console - > print ( " Loading config \n " ) ;
auto askPassword = [ console ] ( ) {
return console - > askPassword ( " Password: " ) ;
} ;
2018-11-10 21:38:57 +01:00
unique_ref < CryKeyProvider > keyProvider = make_unique_ref < CryPasswordBasedKeyProvider > (
2018-10-22 04:31:08 +02:00
console ,
askPassword ,
askPassword ,
make_unique_ref < SCrypt > ( SCrypt : : DefaultSettings )
) ;
2018-12-22 01:22:24 +01:00
2018-11-10 21:38:57 +01:00
auto config_path = basedir / " cryfs.config " ;
LocalStateDir localStateDir ( cpputils : : system : : HomeDirectory : : getXDGDataDir ( ) / " cryfs " ) ;
CryConfigLoader config_loader ( console , Random : : OSRandom ( ) , std : : move ( keyProvider ) , localStateDir , boost : : none , boost : : none , boost : : none ) ;
auto config = config_loader . load ( config_path , false , true ) ;
if ( config = = boost : : none ) {
2018-12-22 01:22:24 +01:00
// TODO Show more info about error
throw std : : runtime_error ( " Error loading config file. " ) ;
2018-11-10 21:38:57 +01:00
}
2019-01-31 00:57:13 +01:00
const auto & config_ = config - > configFile - > config ( ) ;
2019-01-22 07:53:28 +01:00
std : : cout < < " Loading filesystem of version " < < config_ - > Version ( ) < < std : : endl ;
# ifndef CRYFS_NO_COMPATIBILITY
2019-05-26 11:34:26 +02:00
const bool is_correct_format = config_ - > Version ( ) = = CryConfig : : FilesystemFormatVersion & & config_ - > HasParentPointers ( ) & & config_ - > HasVersionNumbers ( ) ;
2019-01-22 07:53:28 +01:00
# else
const bool is_correct_format = config_ - > Version ( ) = = CryConfig : : FilesystemFormatVersion ;
# endif
if ( ! is_correct_format ) {
// TODO At this point, the cryfs.config file was already switched to 0.10 format. We should probably not do that.
std : : cerr < < " The filesystem is not in the 0.10 format. It needs to be migrated. The cryfs-stats tool unfortunately can't handle this, please mount and unmount the filesystem once. " < < std : : endl ;
exit ( 1 ) ;
}
2018-11-10 21:38:57 +01:00
cout < < " Listing all blocks... " < < flush ;
set < BlockId > unaccountedBlocks = _getAllBlockIds ( basedir , * config , localStateDir ) ;
cout < < " done " < < endl ;
vector < BlockId > accountedBlocks = _getKnownBlockIds ( basedir , * config , localStateDir ) ;
for ( const BlockId & blockId : accountedBlocks ) {
auto num_erased = unaccountedBlocks . erase ( blockId ) ;
ASSERT ( 1 = = num_erased , " Blob id referenced by directory entry but didn't found it on disk? This can't happen. " ) ;
2016-03-01 17:45:48 +01:00
}
2018-10-22 04:31:08 +02:00
console - > print ( " Calculate statistics \n " ) ;
2016-03-01 17:45:48 +01:00
2018-11-10 21:38:57 +01:00
auto blockStore = makeBlockStore ( basedir , * config , localStateDir ) ;
2018-12-22 00:58:30 +01:00
auto nodeStore = make_unique_ref < DataNodeStore > ( std : : move ( blockStore ) , config - > configFile - > config ( ) - > BlocksizeBytes ( ) ) ;
2016-03-01 17:45:48 +01:00
uint32_t numUnaccountedBlocks = unaccountedBlocks . size ( ) ;
uint32_t numLeaves = 0 ;
uint32_t numInner = 0 ;
2018-10-22 04:31:08 +02:00
console - > print ( " Unaccounted blocks: " + std : : to_string ( unaccountedBlocks . size ( ) ) + " \n " ) ;
2017-09-17 03:07:27 +02:00
for ( const auto & blockId : unaccountedBlocks ) {
2018-11-10 21:38:57 +01:00
console - > print ( " \r " + std : : to_string ( numLeaves + numInner ) + " / " + std : : to_string ( numUnaccountedBlocks ) + " : " ) ;
2017-09-17 03:07:27 +02:00
auto node = nodeStore - > load ( blockId ) ;
2016-03-01 17:45:48 +01:00
auto innerNode = dynamic_pointer_move < DataInnerNode > ( * node ) ;
if ( innerNode ! = none ) {
+ + numInner ;
printNode ( std : : move ( * innerNode ) ) ;
}
auto leafNode = dynamic_pointer_move < DataLeafNode > ( * node ) ;
if ( leafNode ! = none ) {
+ + numLeaves ;
printNode ( std : : move ( * leafNode ) ) ;
}
}
2018-10-22 04:31:08 +02:00
console - > print ( " \n " + std : : to_string ( numLeaves ) + " leaves and " + std : : to_string ( numInner ) + " inner nodes \n " ) ;
2017-09-09 15:44:01 +02:00
}